From 27687475265f88bc0a0bcbfe2ba26da306bbfc20 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 10 Apr 2025 09:43:32 +0100 Subject: [PATCH 1/3] Fixed #36315 -- Used TaskGroup instead of asyncio.gather(). --- django/dispatch/dispatcher.py | 42 ++++++++++++++++++++++++++--------- docs/topics/signals.txt | 6 ++++- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/django/dispatch/dispatcher.py b/django/dispatch/dispatcher.py index 120f2ac6dead..63fb75285ec3 100644 --- a/django/dispatch/dispatcher.py +++ b/django/dispatch/dispatcher.py @@ -22,6 +22,28 @@ def _make_id(target): NO_RECEIVERS = object() +async def _gather(*coros): + if len(coros) == 0: + return [] + + if len(coros) == 1: + return [await coros[0]] + + async def run(i, coro): + results[i] = await coro + + try: + async with asyncio.TaskGroup() as tg: + results = [None] * len(coros) + for i, coro in enumerate(coros): + tg.create_task(run(i, coro)) + return results + except BaseExceptionGroup as exception_group: + if len(exception_group.exceptions) == 1: + raise exception_group.exceptions[0] + raise + + class Signal: """ Base class for all signals @@ -186,7 +208,7 @@ def send(self, sender, **named): If any receivers are asynchronous, they are called after all the synchronous receivers via a single call to async_to_sync(). They are - also executed concurrently with asyncio.gather(). + also executed concurrently with asyncio.TaskGroup(). Arguments: @@ -211,7 +233,7 @@ def send(self, sender, **named): if async_receivers: async def asend(): - async_responses = await asyncio.gather( + async_responses = await _gather( *( receiver(signal=self, sender=sender, **named) for receiver in async_receivers @@ -235,7 +257,7 @@ async def asend(self, sender, **named): sync_to_async() adaption before executing any asynchronous receivers. If any receivers are asynchronous, they are grouped and executed - concurrently with asyncio.gather(). + concurrently with asyncio.TaskGroup(). Arguments: @@ -268,9 +290,9 @@ def sync_send(): async def sync_send(): return [] - responses, async_responses = await asyncio.gather( + responses, async_responses = await _gather( sync_send(), - asyncio.gather( + _gather( *( receiver(signal=self, sender=sender, **named) for receiver in async_receivers @@ -294,7 +316,7 @@ def send_robust(self, sender, **named): If any receivers are asynchronous, they are called after all the synchronous receivers via a single call to async_to_sync(). They are - also executed concurrently with asyncio.gather(). + also executed concurrently with asyncio.TaskGroup(). Arguments: @@ -340,7 +362,7 @@ async def asend_and_wrap_exception(receiver): return response async def asend(): - async_responses = await asyncio.gather( + async_responses = await _gather( *( asend_and_wrap_exception(receiver) for receiver in async_receivers @@ -359,7 +381,7 @@ async def asend_robust(self, sender, **named): sync_to_async() adaption before executing any asynchronous receivers. If any receivers are asynchronous, they are grouped and executed - concurrently with asyncio.gather. + concurrently with asyncio.TaskGroup. Arguments: @@ -414,9 +436,9 @@ async def asend_and_wrap_exception(receiver): return err return response - responses, async_responses = await asyncio.gather( + responses, async_responses = await _gather( sync_send(), - asyncio.gather( + _gather( *(asend_and_wrap_exception(receiver) for receiver in async_receivers), ), ) diff --git a/docs/topics/signals.txt b/docs/topics/signals.txt index 4d32c55a2b5c..8f2246c55a30 100644 --- a/docs/topics/signals.txt +++ b/docs/topics/signals.txt @@ -323,11 +323,15 @@ in order to reduce the number of sync/async calling-style switches within a they are async before being called. This means that an asynchronous receiver registered before a synchronous receiver may be executed after the synchronous receiver. In addition, async receivers are executed concurrently using -``asyncio.gather()``. +:class:`asyncio.TaskGroup`. All built-in signals, except those in the async request-response cycle, are dispatched using :meth:`Signal.send`. +.. versionchanged:: 6.1 + + In older versions, async receivers were executed via ``asyncio.gather()``. + Disconnecting signals ===================== From 1c7db70e79dce82f50d5958da64ab8e2807a31df Mon Sep 17 00:00:00 2001 From: Kasyap Pentamaraju Date: Mon, 3 Nov 2025 22:57:53 +0530 Subject: [PATCH 2/3] Fixed #36705 -- Avoided string concatenation in utils. Repeated string concatenation performs poorly on PyPy. Thanks Seokchan Yoon for the report. --- django/utils/html.py | 12 +++++++----- django/utils/http.py | 6 +++--- django/utils/numberformat.py | 8 ++++---- django/utils/text.py | 20 ++++++++++---------- docs/topics/performance.txt | 6 ++++++ 5 files changed, 30 insertions(+), 22 deletions(-) diff --git a/django/utils/html.py b/django/utils/html.py index b04d6bc5eef6..059767d394fd 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -4,6 +4,7 @@ import json import re import warnings +from collections import deque from collections.abc import Mapping from html.parser import HTMLParser from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit, urlunsplit @@ -429,7 +430,7 @@ def trim_punctuation(self, word): # Strip all opening wrapping punctuation. middle = word.lstrip(self.wrapping_punctuation_openings) lead = word[: len(word) - len(middle)] - trail = "" + trail = deque() # Continue trimming until middle remains unchanged. trimmed_something = True @@ -442,7 +443,7 @@ def trim_punctuation(self, word): rstripped = middle.rstrip(closing) if rstripped != middle: strip = counts[closing] - counts[opening] - trail = middle[-strip:] + trail.appendleft(middle[-strip:]) middle = middle[:-strip] trimmed_something = True counts[closing] -= strip @@ -453,7 +454,7 @@ def trim_punctuation(self, word): else: rstripped = middle.rstrip(self.trailing_punctuation_chars_no_semicolon) if rstripped != middle: - trail = middle[len(rstripped) :] + trail + trail.appendleft(middle[len(rstripped) :]) middle = rstripped trimmed_something = True @@ -470,13 +471,14 @@ def trim_punctuation(self, word): # entity. recent_semicolon = middle[trail_start:].index(";") middle_semicolon_index = recent_semicolon + trail_start + 1 - trail = middle[middle_semicolon_index:] + trail + trail.appendleft(middle[middle_semicolon_index:]) middle = rstripped + middle[trail_start:middle_semicolon_index] else: - trail = middle[trail_start:] + trail + trail.appendleft(middle[trail_start:]) middle = rstripped trimmed_something = True + trail = "".join(trail) return lead, middle, trail @staticmethod diff --git a/django/utils/http.py b/django/utils/http.py index fe0b21f150a8..21d5822bf291 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -169,11 +169,11 @@ def int_to_base36(i): raise ValueError("Negative base36 conversion input.") if i < 36: return char_set[i] - b36 = "" + b36_parts = [] while i != 0: i, n = divmod(i, 36) - b36 = char_set[n] + b36 - return b36 + b36_parts.append(char_set[n]) + return "".join(reversed(b36_parts)) def urlsafe_base64_encode(s): diff --git a/django/utils/numberformat.py b/django/utils/numberformat.py index cf8b2d219cc5..f621ff64b494 100644 --- a/django/utils/numberformat.py +++ b/django/utils/numberformat.py @@ -91,15 +91,15 @@ def format( # grouping is a single value intervals = [grouping, 0] active_interval = intervals.pop(0) - int_part_gd = "" + int_part_gd = [] cnt = 0 for digit in int_part[::-1]: if cnt and cnt == active_interval: if intervals: active_interval = intervals.pop(0) or active_interval - int_part_gd += thousand_sep[::-1] + int_part_gd.append(thousand_sep[::-1]) cnt = 0 - int_part_gd += digit + int_part_gd.append(digit) cnt += 1 - int_part = int_part_gd[::-1] + int_part = "".join(int_part_gd)[::-1] return sign + int_part + dec_part diff --git a/django/utils/text.py b/django/utils/text.py index bad1da672941..baf44265a49c 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -103,7 +103,7 @@ class TruncationCompleted(Exception): def __init__(self, *, length, replacement, convert_charrefs=True): super().__init__(convert_charrefs=convert_charrefs) self.tags = deque() - self.output = "" + self.output = [] self.remaining = length self.replacement = replacement @@ -119,13 +119,13 @@ def handle_startendtag(self, tag, attrs): self.handle_endtag(tag) def handle_starttag(self, tag, attrs): - self.output += self.get_starttag_text() + self.output.append(self.get_starttag_text()) if tag not in self.void_elements: self.tags.appendleft(tag) def handle_endtag(self, tag): if tag not in self.void_elements: - self.output += f"" + self.output.append(f"") try: self.tags.remove(tag) except ValueError: @@ -136,16 +136,16 @@ def handle_data(self, data): data_len = len(data) if self.remaining < data_len: self.remaining = 0 - self.output += add_truncation_text(output, self.replacement) + self.output.append(add_truncation_text(output, self.replacement)) raise self.TruncationCompleted self.remaining -= data_len - self.output += output + self.output.append(output) def feed(self, data): try: super().feed(data) except self.TruncationCompleted: - self.output += "".join([f"" for tag in self.tags]) + self.output.extend([f"" for tag in self.tags]) self.tags.clear() self.reset() else: @@ -166,9 +166,9 @@ def __init__(self, *, length, replacement, convert_charrefs=True): def process(self, data): self.processed_chars += len(data) if (self.processed_chars == self.length) and ( - len(self.output) + len(data) == len(self.rawdata) + sum(len(p) for p in self.output) + len(data) == len(self.rawdata) ): - self.output += data + self.output.append(data) raise self.TruncationCompleted output = escape("".join(data[: self.remaining])) return data, output @@ -213,7 +213,7 @@ def chars(self, num, truncate=None, html=False): parser = TruncateCharsHTMLParser(length=length, replacement=truncate) parser.feed(text) parser.close() - return parser.output + return "".join(parser.output) return self._text_chars(length, truncate, text) def _text_chars(self, length, truncate, text): @@ -250,7 +250,7 @@ def words(self, num, truncate=None, html=False): parser = TruncateWordsHTMLParser(length=length, replacement=truncate) parser.feed(self._wrapped) parser.close() - return parser.output + return "".join(parser.output) return self._text_words(length, truncate) def _text_words(self, length, truncate): diff --git a/docs/topics/performance.txt b/docs/topics/performance.txt index dcffb1a68317..2935f21f67de 100644 --- a/docs/topics/performance.txt +++ b/docs/topics/performance.txt @@ -426,6 +426,12 @@ Django is compatible with versions of PyPy corresponding to the supported Python versions, but you will need to check the compatibility of other libraries you rely on. +That said, a lot of a web framework's work is done by concatenating +strings, and PyPy has an issue with that (see +`this PyPy blog +`_). +This may cause performance issues, depending on your use. + C implementations of Python libraries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From c4e07f94ebc1f9eaa3dae7b3dc6a2b9832182a10 Mon Sep 17 00:00:00 2001 From: myoungjinGo Date: Wed, 16 Jul 2025 00:43:45 +0900 Subject: [PATCH 3/3] Fixed #36420 -- Used actual SQLite limits in last_executed_query() quoting. --- django/db/backends/sqlite3/operations.py | 18 ++++++------- tests/backends/sqlite/tests.py | 32 +++++++++++++++++------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/django/db/backends/sqlite3/operations.py b/django/db/backends/sqlite3/operations.py index 1500ae28aa0d..ac98324b2e30 100644 --- a/django/db/backends/sqlite3/operations.py +++ b/django/db/backends/sqlite3/operations.py @@ -1,5 +1,6 @@ import datetime import decimal +import sqlite3 import uuid from functools import lru_cache from itertools import chain @@ -143,16 +144,15 @@ def _quote_params_for_last_executed_query(self, params): """ Only for last_executed_query! Don't use this to execute SQL queries! """ - # This function is limited both by SQLITE_LIMIT_VARIABLE_NUMBER (the - # number of parameters, default = 999) and SQLITE_MAX_COLUMN (the - # number of return values, default = 2000). Since Python's sqlite3 - # module doesn't expose the get_limit() C API, assume the default - # limits are in effect and split the work in batches if needed. - BATCH_SIZE = 999 - if len(params) > BATCH_SIZE: + connection = self.connection.connection + variable_limit = self.connection.features.max_query_params + column_limit = connection.getlimit(sqlite3.SQLITE_LIMIT_COLUMN) + batch_size = min(variable_limit, column_limit) + + if len(params) > batch_size: results = () - for index in range(0, len(params), BATCH_SIZE): - chunk = params[index : index + BATCH_SIZE] + for index in range(0, len(params), batch_size): + chunk = params[index : index + batch_size] results += self._quote_params_for_last_executed_query(chunk) return results diff --git a/tests/backends/sqlite/tests.py b/tests/backends/sqlite/tests.py index 37d95c0cb56e..34c0eca0ff41 100644 --- a/tests/backends/sqlite/tests.py +++ b/tests/backends/sqlite/tests.py @@ -1,5 +1,6 @@ import os import re +import sqlite3 import tempfile import threading import unittest @@ -215,15 +216,28 @@ def test_parameter_quoting(self): substituted = "SELECT '\"''\\'" self.assertEqual(connection.queries[-1]["sql"], substituted) - def test_large_number_of_parameters(self): - # If SQLITE_MAX_VARIABLE_NUMBER (default = 999) has been changed to be - # greater than SQLITE_MAX_COLUMN (default = 2000), last_executed_query - # can hit the SQLITE_MAX_COLUMN limit (#26063). - with connection.cursor() as cursor: - sql = "SELECT MAX(%s)" % ", ".join(["%s"] * 2001) - params = list(range(2001)) - # This should not raise an exception. - cursor.db.ops.last_executed_query(cursor.cursor, sql, params) + def test_parameter_count_exceeds_variable_or_column_limit(self): + sql = "SELECT MAX(%s)" % ", ".join(["%s"] * 1001) + params = list(range(1001)) + for label, limit, current_limit in [ + ( + "variable", + sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, + connection.features.max_query_params, + ), + ( + "column", + sqlite3.SQLITE_LIMIT_COLUMN, + connection.connection.getlimit(sqlite3.SQLITE_LIMIT_COLUMN), + ), + ]: + with self.subTest(limit=label): + connection.connection.setlimit(limit, 1000) + self.addCleanup(connection.connection.setlimit, limit, current_limit) + with connection.cursor() as cursor: + # This should not raise an exception. + cursor.db.ops.last_executed_query(cursor.cursor, sql, params) + connection.connection.setlimit(limit, current_limit) @unittest.skipUnless(connection.vendor == "sqlite", "SQLite tests")