Skip to content

Commit 905c80b

Browse files
authored
Merge branch 'main' into pyslot_end
2 parents c3671d7 + eab2d16 commit 905c80b

76 files changed

Lines changed: 958 additions & 1341 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/reusable-san.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ jobs:
7272
- name: MSan option setup
7373
if: inputs.sanitizer == 'MSan'
7474
run: |
75+
sudo sysctl -w vm.mmap_rnd_bits=28 # Reduce ASLR to avoid MSan re-executing
76+
7577
echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV"
7678
# MSan reports false positives for memory initialized by libraries
7779
# that are not built with MSan, so disable modules that use them.

‎Include/internal/pycore_call.h‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,6 @@ extern PyObject* _PyObject_Call(
5757
PyObject *args,
5858
PyObject *kwargs);
5959

60-
extern PyObject * _PyObject_CallMethodFormat(
61-
PyThreadState *tstate,
62-
PyObject *callable,
63-
const char *format,
64-
...);
65-
6660
// Export for 'array' shared extension
6761
PyAPI_FUNC(PyObject*) _PyObject_CallMethod(
6862
PyObject *obj,

‎Include/internal/pycore_unicodeobject.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,7 @@ extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **);
468468
extern void _PyUnicode_ClearInterned(PyInterpreterState *interp);
469469

470470
// Like PyUnicode_AsUTF8(), but check for embedded null characters.
471-
// Export for '_sqlite3' shared extension.
471+
// Export for '_sqlite3' shared extension, and for Argument Clinic code.
472472
PyAPI_FUNC(const char *) _PyUnicode_AsUTF8NoNUL(PyObject *);
473473

474474

‎Lib/profiling/sampling/heatmap_collector.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -783,14 +783,14 @@ def _generate_file_html(self, output_path: Path, filename: str,
783783
line_counts: Dict[int, int], self_counts: Dict[int, int],
784784
file_stat: FileStats):
785785
"""Generate HTML for a single source file with heatmap coloring."""
786-
# Read source file
786+
source_lines = [f"# Source file not available: {filename}"]
787787
try:
788-
source_lines = Path(filename).read_text(encoding='utf-8', errors='replace').splitlines()
789-
except (IOError, OSError) as e:
790-
if not (filename.startswith('<') or filename.startswith('[') or
791-
filename in ('~', '...', '.') or len(filename) < 2):
792-
print(f"Warning: Could not read source file {filename}: {e}")
793-
source_lines = [f"# Source file not available: {filename}"]
788+
path = Path(filename)
789+
if path.is_file():
790+
source_lines = path.read_text(
791+
encoding='utf-8', errors='replace').splitlines()
792+
except (IOError, OSError):
793+
pass
794794

795795
# Generate HTML for each line
796796
max_samples = max(line_counts.values()) if line_counts else 1

‎Lib/profiling/sampling/stack_collector.py‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ def export(self, filename):
6868
return True
6969

7070

71-
# Bounded by the unwinder's maximum captured stack depth (MAX_FRAMES).
72-
_FLAMEGRAPH_RECURSION_MARGIN = 2000
71+
# Allow for tree conversion and the dict/list frames in the Python JSON encoder.
72+
_FLAMEGRAPH_RECURSION_MARGIN = 6000
7373

7474

7575
class FlamegraphCollector(StackTraceCollector):
@@ -498,7 +498,12 @@ def _get_source_lines(self, func):
498498
return None
499499

500500
def _create_flamegraph_html(self, data):
501-
data_json = json.dumps(data)
501+
try:
502+
data_json = json.dumps(data)
503+
except RecursionError:
504+
# The C encoder can exhaust the C stack independently of the
505+
# Python recursion limit. iterencode() uses the Python encoder.
506+
data_json = "".join(json.JSONEncoder().iterencode(data))
502507

503508
template_dir = importlib.resources.files(__package__)
504509
vendor_dir = template_dir / "_vendor"
@@ -676,16 +681,16 @@ def _convert_to_flamegraph_format(self):
676681
current_stats = self._aggregate_path_samples(self._root)
677682
baseline_stats = self._aggregate_path_samples(self._baseline_collector._root)
678683

679-
# Scale baseline values to make them comparable, accounting for both
680-
# sample count differences and sample interval differences.
684+
# Express baseline samples in units of the current sample interval.
685+
# Do not normalize by total profile duration: doing so makes unchanged
686+
# functions appear different when another function becomes faster or
687+
# slower.
681688
baseline_total = self._baseline_collector._total_samples
682-
if baseline_total > 0 and self._total_samples > 0:
683-
current_time = self._total_samples * self.sample_interval_usec
684-
baseline_time = baseline_total * self._baseline_collector.sample_interval_usec
685-
scale = current_time / baseline_time
686-
elif baseline_total > 0:
687-
# Current profile is empty - use interval-based scale for elided display
688-
scale = self.sample_interval_usec / self._baseline_collector.sample_interval_usec
689+
if baseline_total > 0:
690+
scale = (
691+
self._baseline_collector.sample_interval_usec
692+
/ self.sample_interval_usec
693+
)
689694
else:
690695
scale = 1.0
691696

@@ -902,6 +907,10 @@ def _add_elided_metadata(self, node, baseline_stats, scale, path):
902907
else:
903908
node["diff_pct"] = 0.0
904909

910+
# Scale geometry after computing metadata from raw baseline counts.
911+
node["value"] = node.get("value", 0) * scale
912+
node["self"] = node.get("self", 0) * scale
913+
905914
if "children" in node and node["children"]:
906915
for child in node["children"]:
907916
self._add_elided_metadata(child, baseline_stats, scale, current_path)

‎Lib/tempfile.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,11 +119,15 @@ def _sanitize_params(prefix, suffix, dir):
119119
output_type = _infer_return_type(prefix, suffix, dir)
120120
if suffix is None:
121121
suffix = output_type()
122+
if _os.path.dirname(suffix):
123+
raise ValueError("suffix can't contain a directory component")
122124
if prefix is None:
123125
if output_type is str:
124126
prefix = template
125127
else:
126128
prefix = _os.fsencode(template)
129+
if _os.path.dirname(prefix):
130+
raise ValueError("prefix can't contain a directory component")
127131
if dir is None:
128132
if output_type is str:
129133
dir = gettempdir()

‎Lib/test/test_external_inspection.py‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,22 @@ async def main():
492492
self.assertIn(main_name, names)
493493
self.assertEqual([len(n) for n in names if n.startswith("x")], [255])
494494

495+
@skip_if_not_supported
496+
def test_recursive_coroutine_stack_is_not_truncated(self):
497+
# gh-158522
498+
async def rec(n):
499+
if n:
500+
return await rec(n - 1)
501+
return [
502+
frame.funcname.rpartition(".")[2]
503+
for task in RemoteUnwinder(
504+
os.getpid()).get_async_stack_trace()[0].awaited_by
505+
for coro in task.coroutine_stack
506+
for frame in coro.call_stack
507+
]
508+
509+
self.assertEqual(asyncio.run(rec(3)), ["rec"] * 4)
510+
495511
@skip_if_not_supported
496512
@unittest.skipIf(
497513
sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
@@ -3293,6 +3309,242 @@ def test_finally_no_exception_no_flag(self):
32933309
self._check_exception_status(p, thread_tid, expect_exception=False)
32943310

32953311

3312+
@skip_if_not_supported
3313+
class TestExceptionDetectionInProcess(RemoteInspectionTestBase):
3314+
"""gh-158539: HAS_EXCEPTION for handlers running in generators/coroutines.
3315+
3316+
``TestExceptionDetectionScenarios`` samples a child process and therefore
3317+
needs subprocess debugging permissions. These tests inspect the current
3318+
process with ``RemoteUnwinder`` and only need self-inspection, so they also
3319+
run on macOS without special entitlements.
3320+
"""
3321+
3322+
@classmethod
3323+
def setUpClass(cls):
3324+
try:
3325+
RemoteUnwinder(os.getpid(), all_threads=True).get_stack_trace()
3326+
except PermissionError as exc:
3327+
raise unittest.SkipTest(f"self-inspection is unavailable: {exc}")
3328+
3329+
def _check_running_handler(
3330+
self, target, expect_exception, *, mode=PROFILING_MODE_ALL,
3331+
skip_non_matching_threads=False,
3332+
):
3333+
"""Run *target* in a thread and check its HAS_EXCEPTION flag.
3334+
3335+
*target* receives ``(ready, stop)`` events and must signal ``ready``
3336+
only once it is executing inside the code region under test, then keep
3337+
running until ``stop`` is set.
3338+
"""
3339+
stop = threading.Event()
3340+
ready = threading.Event()
3341+
failure = []
3342+
3343+
def runner():
3344+
try:
3345+
target(ready, stop)
3346+
except BaseException as exc:
3347+
failure.append(exc)
3348+
ready.set()
3349+
3350+
thread = threading.Thread(target=runner, daemon=True)
3351+
thread.start()
3352+
try:
3353+
self.assertTrue(ready.wait(SHORT_TIMEOUT), "handler never started")
3354+
self.assertFalse(failure, f"handler raised {failure!r}")
3355+
3356+
unwinder = RemoteUnwinder(
3357+
os.getpid(),
3358+
all_threads=True,
3359+
mode=mode,
3360+
skip_non_matching_threads=skip_non_matching_threads,
3361+
)
3362+
observed = []
3363+
for _ in busy_retry(SHORT_TIMEOUT):
3364+
with contextlib.suppress(*TRANSIENT_ERRORS):
3365+
statuses = self._get_thread_statuses(unwinder.get_stack_trace())
3366+
status = statuses.get(thread.native_id)
3367+
if status is None:
3368+
continue
3369+
has_exception = bool(status & THREAD_STATUS_HAS_EXCEPTION)
3370+
observed.append(has_exception)
3371+
if has_exception == expect_exception:
3372+
break
3373+
self.assertTrue(
3374+
observed, "target thread status was never observed"
3375+
)
3376+
self.assertIn(
3377+
expect_exception,
3378+
observed,
3379+
f"HAS_EXCEPTION was never {expect_exception} while the "
3380+
f"handler was running (observed {observed})",
3381+
)
3382+
finally:
3383+
stop.set()
3384+
thread.join(SHORT_TIMEOUT)
3385+
3386+
def _busy_until_stopped(self, ready, stop):
3387+
ready.set()
3388+
while not stop.is_set():
3389+
time.sleep(0.001)
3390+
3391+
def test_handler_in_function(self):
3392+
def target(ready, stop):
3393+
try:
3394+
raise ValueError("test")
3395+
except ValueError:
3396+
self._busy_until_stopped(ready, stop)
3397+
3398+
self._check_running_handler(target, expect_exception=True)
3399+
3400+
def test_handler_in_generator(self):
3401+
def target(ready, stop):
3402+
def gen():
3403+
try:
3404+
raise ValueError("test")
3405+
except ValueError:
3406+
self._busy_until_stopped(ready, stop)
3407+
yield
3408+
3409+
for _ in gen():
3410+
pass
3411+
3412+
self._check_running_handler(target, expect_exception=True)
3413+
3414+
def test_handler_in_genexpr_callee(self):
3415+
def target(ready, stop):
3416+
def callee():
3417+
try:
3418+
raise ValueError("test")
3419+
except ValueError:
3420+
self._busy_until_stopped(ready, stop)
3421+
3422+
list(callee() for _ in range(1))
3423+
3424+
self._check_running_handler(target, expect_exception=True)
3425+
3426+
def test_handler_in_coroutine(self):
3427+
async def coro(ready, stop):
3428+
try:
3429+
raise ValueError("test")
3430+
except ValueError:
3431+
self._busy_until_stopped(ready, stop)
3432+
3433+
def target(ready, stop):
3434+
asyncio.run(coro(ready, stop))
3435+
3436+
self._check_running_handler(target, expect_exception=True)
3437+
3438+
def test_handler_in_callee_from_coroutine(self):
3439+
def callee(ready, stop):
3440+
try:
3441+
raise ValueError("test")
3442+
except ValueError:
3443+
self._busy_until_stopped(ready, stop)
3444+
3445+
async def coro(ready, stop):
3446+
callee(ready, stop)
3447+
3448+
def target(ready, stop):
3449+
asyncio.run(coro(ready, stop))
3450+
3451+
self._check_running_handler(target, expect_exception=True)
3452+
3453+
def test_outer_handler_while_generator_runs(self):
3454+
"""A generator with no handler of its own must not hide the outer one.
3455+
3456+
``exc_info`` points at the generator's empty ``_PyErr_StackItem`` whose
3457+
``previous_item`` is the thread's ``exc_state``, so the profiler has to
3458+
walk the chain to find the exception ``sys.exception()`` reports.
3459+
"""
3460+
def target(ready, stop):
3461+
def gen():
3462+
self._busy_until_stopped(ready, stop)
3463+
yield
3464+
3465+
try:
3466+
raise ValueError("outer")
3467+
except ValueError:
3468+
for _ in gen():
3469+
pass
3470+
3471+
self._check_running_handler(target, expect_exception=True)
3472+
3473+
def test_generator_without_exception(self):
3474+
def target(ready, stop):
3475+
def gen():
3476+
self._busy_until_stopped(ready, stop)
3477+
yield
3478+
3479+
for _ in gen():
3480+
pass
3481+
3482+
self._check_running_handler(target, expect_exception=False)
3483+
3484+
def test_outer_handler_while_nested_generators_run(self):
3485+
def target(ready, stop):
3486+
def gen(depth):
3487+
if depth:
3488+
yield from gen(depth - 1)
3489+
else:
3490+
self._busy_until_stopped(ready, stop)
3491+
yield
3492+
3493+
try:
3494+
raise ValueError("outer")
3495+
except ValueError:
3496+
for _ in gen(32):
3497+
pass
3498+
3499+
self._check_running_handler(
3500+
target,
3501+
expect_exception=True,
3502+
mode=PROFILING_MODE_EXCEPTION,
3503+
skip_non_matching_threads=True,
3504+
)
3505+
3506+
def test_generator_finally_after_except(self):
3507+
"""The handled exception is cleared before the generator's finally."""
3508+
def target(ready, stop):
3509+
def gen():
3510+
try:
3511+
raise ValueError("test")
3512+
except ValueError:
3513+
pass
3514+
finally:
3515+
self._busy_until_stopped(ready, stop)
3516+
yield
3517+
3518+
for _ in gen():
3519+
pass
3520+
3521+
self._check_running_handler(target, expect_exception=False)
3522+
3523+
def test_exception_mode_filter_keeps_generator_handler(self):
3524+
"""The exception-mode thread filter must not drop a generator handler.
3525+
3526+
This mirrors what ``--mode=exception`` actually does: threads without
3527+
HAS_EXCEPTION are skipped before their stack is unwound.
3528+
"""
3529+
def target(ready, stop):
3530+
def gen():
3531+
try:
3532+
raise ValueError("test")
3533+
except ValueError:
3534+
self._busy_until_stopped(ready, stop)
3535+
yield
3536+
3537+
for _ in gen():
3538+
pass
3539+
3540+
self._check_running_handler(
3541+
target,
3542+
expect_exception=True,
3543+
mode=PROFILING_MODE_EXCEPTION,
3544+
skip_non_matching_threads=True,
3545+
)
3546+
3547+
32963548
@requires_remote_subprocess_debugging()
32973549
class TestFrameCaching(RemoteInspectionTestBase):
32983550
"""Test that frame caching produces correct results.

0 commit comments

Comments
 (0)