GH-126910: Add GNU backtrace support for unwinding JIT frames (#149104)

Co-authored-by: Pablo Galindo Salgado <pablogsal@gmail.com>

Author: diegorusso

Include/internal/pycore_jit_publish.h

@@ -0,0 +1,31 @@
+#ifndef Py_INTERNAL_JIT_PUBLISH_H
+#define Py_INTERNAL_JIT_PUBLISH_H
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include <stddef.h>
+
+typedef struct _PyJitCodeRegistration _PyJitCodeRegistration;
+
+#ifdef _Py_JIT
+
+/* Publish JIT code to optional tooling backends.
+ *
+ * The return value is a backend-specific deregistration handle, not a
+ * success/failure indicator. NULL means there is nothing to unregister later:
+ * perf does not need a handle, and GDB/GNU backtrace registration failures
+ * are intentionally non-fatal because tooling support must not make JIT
+ * compilation fail.
+ */
+_PyJitCodeRegistration *_PyJit_RegisterCode(const void *code_addr,
+                                            size_t code_size,
+                                            const char *entry,
+                                            const char *filename);
+
+void _PyJit_UnregisterCode(_PyJitCodeRegistration *registration);
+
+#endif  // _Py_JIT
+
+#endif  // Py_INTERNAL_JIT_PUBLISH_H

Include/internal/pycore_jit_unwind.h

@@ -10,9 +10,15 @@
 
 #if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__)
 #  define PY_HAVE_JIT_GDB_UNWIND
+#  if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
+          defined(HAVE_LIBGCC_EH_FRAME_REGISTRATION)
+#    define PY_HAVE_JIT_GNU_BACKTRACE_UNWIND
+#  endif
 #endif
 
-#if defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND)
+#if defined(PY_HAVE_PERF_TRAMPOLINE) \
+    || defined(PY_HAVE_JIT_GDB_UNWIND) \
+    || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND)
 
 #if defined(PY_HAVE_JIT_GDB_UNWIND)
 extern PyMutex _Py_jit_debug_mutex;
@@ -63,6 +69,13 @@ void *_PyJitUnwind_GdbRegisterCode(const void *code_addr,
 
 void _PyJitUnwind_GdbUnregisterCode(void *handle);
 
-#endif  // defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND)
+#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND)
+void *_PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr,
+                                            size_t code_size);
+
+void _PyJitUnwind_GnuBacktraceUnregisterCode(void *handle);
+#endif
+
+#endif  // JIT unwind support
 
 #endif  // Py_INTERNAL_JIT_UNWIND_H

Include/internal/pycore_optimizer.h

@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_typedefs.h"      // _PyInterpreterFrame
+#include "pycore_jit_publish.h"
 #include "pycore_uop.h"           // _PyUOpInstruction
 #include "pycore_uop_ids.h"
 #include "pycore_stackref.h"      // _PyStackRef
@@ -198,7 +199,7 @@ typedef struct _PyExecutorObject {
     uint32_t code_size;
     size_t jit_size;
     void *jit_code;
-    void *jit_gdb_handle;
+    _PyJitCodeRegistration *jit_registration;
     _PyExitData exits[1];
 } _PyExecutorObject;
 

Lib/test/test_frame_pointer_unwind.py

@@ -17,6 +17,9 @@
     raise unittest.SkipTest("test requires subprocess support")
 
 
+STACK_DEPTH = 10
+
+
 def _frame_pointers_expected(machine):
     cflags = " ".join(
         value for value in (
@@ -70,7 +73,7 @@ def _frame_pointers_expected(machine):
     return None
 
 
-def _build_stack_and_unwind():
+def _build_stack_and_unwind(unwinder):
     import operator
 
     def build_stack(n, unwinder, warming_up_caller=False):
@@ -89,7 +92,7 @@ def build_stack(n, unwinder, warming_up_caller=False):
             result = operator.call(build_stack, n - 1, unwinder, warming_up)
         return result
 
-    stack = build_stack(10, _testinternalcapi.manual_frame_pointer_unwind)
+    stack = build_stack(STACK_DEPTH, unwinder)
     return stack
 
 
@@ -112,8 +115,7 @@ def _classify_stack(stack, jit_enabled):
     return annotated, python_frames, jit_frames, other_frames
 
 
-def _annotate_unwind():
-    stack = _build_stack_and_unwind()
+def _summarize_unwind(stack, unwinder_name):
     jit_enabled = hasattr(sys, "_jit") and sys._jit.is_enabled()
     jit_backend = _testinternalcapi.get_jit_backend()
     ranges = _testinternalcapi.get_jit_code_ranges() if jit_enabled else []
@@ -126,19 +128,44 @@ def _annotate_unwind():
     )
     for idx, addr, tag in annotated:
         print(f"#{idx:02d} {addr:#x} -> {tag}")
-    return json.dumps({
+    return {
         "length": len(stack),
         "python_frames": python_frames,
         "jit_frames": jit_frames,
         "other_frames": other_frames,
         "jit_backend": jit_backend,
+        "unwinder": unwinder_name,
+    }
+
+
+def _annotate_unwind(unwinder_name="manual_frame_pointer_unwind"):
+    unwinder = getattr(_testinternalcapi, unwinder_name)
+    stack = _build_stack_and_unwind(unwinder)
+    return json.dumps(_summarize_unwind(stack, unwinder_name))
+
+
+def _annotate_unwind_after_executor_free(unwinder_name="gnu_backtrace_unwind"):
+    # The first unwind runs at the bottom of _build_stack_and_unwind(), while
+    # the recursive helper may be executing in JIT code. After it returns, this
+    # helper is back in normal test code; clearing executor caches should remove
+    # the old JIT ranges, so the second unwind must not report stale JIT frames.
+    live = json.loads(_annotate_unwind(unwinder_name))
+
+    sys._clear_internal_caches()
+    _testinternalcapi.clear_executor_deletion_list()
+
+    unwinder = getattr(_testinternalcapi, unwinder_name)
+    after_free = _summarize_unwind(unwinder(), unwinder_name)
+    return json.dumps({
+        "live": live,
+        "after_free": after_free,
     })
 
 
-def _manual_unwind_length(**env):
+def _run_unwind_helper(helper_name, unwinder_name, **env):
     code = (
-        "from test.test_frame_pointer_unwind import _annotate_unwind; "
-        "print(_annotate_unwind());"
+        f"from test.test_frame_pointer_unwind import {helper_name}; "
+        f"print({helper_name}({unwinder_name!r}));"
     )
     run_env = os.environ.copy()
     run_env.update(env)
@@ -166,6 +193,15 @@ def _manual_unwind_length(**env):
         ) from exc
 
 
+def _unwind_result(unwinder_name, **env):
+    return _run_unwind_helper("_annotate_unwind", unwinder_name, **env)
+
+
+def _unwind_after_executor_free_result(unwinder_name, **env):
+    return _run_unwind_helper(
+        "_annotate_unwind_after_executor_free", unwinder_name, **env)
+
+
 @support.requires_gil_enabled("test requires the GIL enabled")
 @unittest.skipIf(support.is_wasi, "test not supported on WASI")
 class FramePointerUnwindTests(unittest.TestCase):
@@ -197,14 +233,14 @@ def test_manual_unwind_respects_frame_pointers(self):
 
         for env, using_jit in envs:
             with self.subTest(env=env):
-                result = _manual_unwind_length(**env)
+                result = _unwind_result("manual_frame_pointer_unwind", **env)
                 jit_frames = result["jit_frames"]
                 python_frames = result.get("python_frames", 0)
                 jit_backend = result.get("jit_backend")
                 if self.frame_pointers_expected:
-                    self.assertGreater(
+                    self.assertGreaterEqual(
                         python_frames,
-                        0,
+                        STACK_DEPTH,
                         f"expected to find Python frames on {self.machine} with env {env}",
                     )
                     if using_jit:
@@ -240,5 +276,84 @@ def test_manual_unwind_respects_frame_pointers(self):
                     )
 
 
+@support.requires_gil_enabled("test requires the GIL enabled")
+@unittest.skipIf(support.is_wasi, "test not supported on WASI")
+@unittest.skipUnless(sys.platform == "linux", "GNU backtrace unwinding test requires Linux")
+class GnuBacktraceUnwindTests(unittest.TestCase):
+
+    def setUp(self):
+        super().setUp()
+        try:
+            _testinternalcapi.gnu_backtrace_unwind()
+        except RuntimeError as exc:
+            if "not supported" in str(exc):
+                self.skipTest("gnu backtrace unwinding not supported on this platform")
+            raise
+
+    def test_gnu_backtrace_unwinds_through_jit_frames(self):
+        jit_available = hasattr(sys, "_jit") and sys._jit.is_available()
+        envs = [({"PYTHON_JIT": "0"}, False)]
+        if jit_available:
+            envs.append(({"PYTHON_JIT": "1"}, True))
+
+        for env, using_jit in envs:
+            with self.subTest(env=env):
+                result = _unwind_result("gnu_backtrace_unwind", **env)
+                python_frames = result.get("python_frames", 0)
+                jit_frames = result.get("jit_frames", 0)
+                jit_backend = result.get("jit_backend")
+
+                self.assertGreaterEqual(
+                    python_frames,
+                    STACK_DEPTH,
+                    f"expected to find Python frames in GNU backtrace with env {env}",
+                )
+                if using_jit and jit_backend == "jit":
+                    self.assertGreater(
+                        jit_frames,
+                        0,
+                        f"expected GNU backtrace to include JIT frames with env {env}",
+                    )
+                else:
+                    self.assertEqual(
+                        jit_frames,
+                        0,
+                        f"unexpected JIT frames counted in GNU backtrace with env {env}",
+                    )
+
+    def test_gnu_backtrace_jit_frames_disappear_after_executor_free(self):
+        if not (hasattr(sys, "_jit") and sys._jit.is_available()):
+            self.skipTest("JIT is not available")
+
+        result = _unwind_after_executor_free_result(
+            "gnu_backtrace_unwind", PYTHON_JIT="1")
+        live = result["live"]
+        if live.get("jit_backend") != "jit":
+            self.skipTest("JIT backend is not active")
+
+        self.assertGreaterEqual(
+            live.get("python_frames", 0),
+            STACK_DEPTH,
+            "expected live GNU backtrace to include recursive Python frames",
+        )
+        self.assertGreater(
+            live.get("jit_frames", 0),
+            0,
+            "expected live GNU backtrace to include JIT frames",
+        )
+
+        after_free = result["after_free"]
+        self.assertGreater(
+            after_free.get("python_frames", 0),
+            0,
+            "expected GNU backtrace after executor free to include Python frames",
+        )
+        self.assertEqual(
+            after_free.get("jit_frames", 0),
+            0,
+            "unexpected JIT frames in GNU backtrace after executor free",
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

Makefile.pre.in

@@ -470,6 +470,7 @@ PYTHON_OBJS=	\
 		Python/instruction_sequence.o \
 		Python/intrinsics.o \
 		Python/jit.o \
+		Python/jit_publish.o \
 		$(JIT_OBJS) \
 		Python/legacy_tracing.o \
 		Python/lock.o \

Misc/NEWS.d/next/Core_and_Builtins/2026-05-02-18-02-41.gh-issue-126910.nqDVrp.rst

@@ -0,0 +1 @@
+Add support for unwinding JIT frames using GNU backtrace. Patch by Diego Russo and Pablo Galindo