[pull] main from django:main

django/db/backends/sqlite3/operations.py

@@ -1,5 +1,6 @@
 import datetime
 import decimal
+import sqlite3
 import uuid
 from functools import lru_cache
 from itertools import chain
@@ -143,16 +144,15 @@ def _quote_params_for_last_executed_query(self, params):
         """
         Only for last_executed_query! Don't use this to execute SQL queries!
         """
-        # This function is limited both by SQLITE_LIMIT_VARIABLE_NUMBER (the
-        # number of parameters, default = 999) and SQLITE_MAX_COLUMN (the
-        # number of return values, default = 2000). Since Python's sqlite3
-        # module doesn't expose the get_limit() C API, assume the default
-        # limits are in effect and split the work in batches if needed.
-        BATCH_SIZE = 999
-        if len(params) > BATCH_SIZE:
+        connection = self.connection.connection
+        variable_limit = self.connection.features.max_query_params
+        column_limit = connection.getlimit(sqlite3.SQLITE_LIMIT_COLUMN)
+        batch_size = min(variable_limit, column_limit)
+
+        if len(params) > batch_size:
             results = ()
-            for index in range(0, len(params), BATCH_SIZE):
-                chunk = params[index : index + BATCH_SIZE]
+            for index in range(0, len(params), batch_size):
+                chunk = params[index : index + batch_size]
                 results += self._quote_params_for_last_executed_query(chunk)
             return results
 

django/dispatch/dispatcher.py

@@ -22,6 +22,28 @@ def _make_id(target):
 NO_RECEIVERS = object()
 
 
+async def _gather(*coros):
+    if len(coros) == 0:
+        return []
+
+    if len(coros) == 1:
+        return [await coros[0]]
+
+    async def run(i, coro):
+        results[i] = await coro
+
+    try:
+        async with asyncio.TaskGroup() as tg:
+            results = [None] * len(coros)
+            for i, coro in enumerate(coros):
+                tg.create_task(run(i, coro))
+        return results
+    except BaseExceptionGroup as exception_group:
+        if len(exception_group.exceptions) == 1:
+            raise exception_group.exceptions[0]
+        raise
+
+
 class Signal:
     """
     Base class for all signals
@@ -186,7 +208,7 @@ def send(self, sender, **named):
 
         If any receivers are asynchronous, they are called after all the
         synchronous receivers via a single call to async_to_sync(). They are
-        also executed concurrently with asyncio.gather().
+        also executed concurrently with asyncio.TaskGroup().
 
         Arguments:
 
@@ -211,7 +233,7 @@ def send(self, sender, **named):
         if async_receivers:
 
             async def asend():
-                async_responses = await asyncio.gather(
+                async_responses = await _gather(
                     *(
                         receiver(signal=self, sender=sender, **named)
                         for receiver in async_receivers
@@ -235,7 +257,7 @@ async def asend(self, sender, **named):
         sync_to_async() adaption before executing any asynchronous receivers.
 
         If any receivers are asynchronous, they are grouped and executed
-        concurrently with asyncio.gather().
+        concurrently with asyncio.TaskGroup().
 
         Arguments:
 
@@ -268,9 +290,9 @@ def sync_send():
             async def sync_send():
                 return []
 
-        responses, async_responses = await asyncio.gather(
+        responses, async_responses = await _gather(
             sync_send(),
-            asyncio.gather(
+            _gather(
                 *(
                     receiver(signal=self, sender=sender, **named)
                     for receiver in async_receivers
@@ -294,7 +316,7 @@ def send_robust(self, sender, **named):
 
         If any receivers are asynchronous, they are called after all the
         synchronous receivers via a single call to async_to_sync(). They are
-        also executed concurrently with asyncio.gather().
+        also executed concurrently with asyncio.TaskGroup().
 
         Arguments:
 
@@ -340,7 +362,7 @@ async def asend_and_wrap_exception(receiver):
                 return response
 
             async def asend():
-                async_responses = await asyncio.gather(
+                async_responses = await _gather(
                     *(
                         asend_and_wrap_exception(receiver)
                         for receiver in async_receivers
@@ -359,7 +381,7 @@ async def asend_robust(self, sender, **named):
         sync_to_async() adaption before executing any asynchronous receivers.
 
         If any receivers are asynchronous, they are grouped and executed
-        concurrently with asyncio.gather.
+        concurrently with asyncio.TaskGroup.
 
         Arguments:
 
@@ -414,9 +436,9 @@ async def asend_and_wrap_exception(receiver):
                 return err
             return response
 
-        responses, async_responses = await asyncio.gather(
+        responses, async_responses = await _gather(
             sync_send(),
-            asyncio.gather(
+            _gather(
                 *(asend_and_wrap_exception(receiver) for receiver in async_receivers),
             ),
         )

django/utils/html.py

@@ -4,6 +4,7 @@
 import json
 import re
 import warnings
+from collections import deque
 from collections.abc import Mapping
 from html.parser import HTMLParser
 from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit, urlunsplit
@@ -429,7 +430,7 @@ def trim_punctuation(self, word):
         # Strip all opening wrapping punctuation.
         middle = word.lstrip(self.wrapping_punctuation_openings)
         lead = word[: len(word) - len(middle)]
-        trail = ""
+        trail = deque()
 
         # Continue trimming until middle remains unchanged.
         trimmed_something = True
@@ -442,7 +443,7 @@ def trim_punctuation(self, word):
                     rstripped = middle.rstrip(closing)
                     if rstripped != middle:
                         strip = counts[closing] - counts[opening]
-                        trail = middle[-strip:]
+                        trail.appendleft(middle[-strip:])
                         middle = middle[:-strip]
                         trimmed_something = True
                         counts[closing] -= strip
@@ -453,7 +454,7 @@ def trim_punctuation(self, word):
             else:
                 rstripped = middle.rstrip(self.trailing_punctuation_chars_no_semicolon)
             if rstripped != middle:
-                trail = middle[len(rstripped) :] + trail
+                trail.appendleft(middle[len(rstripped) :])
                 middle = rstripped
                 trimmed_something = True
 
@@ -470,13 +471,14 @@ def trim_punctuation(self, word):
                         # entity.
                         recent_semicolon = middle[trail_start:].index(";")
                         middle_semicolon_index = recent_semicolon + trail_start + 1
-                        trail = middle[middle_semicolon_index:] + trail
+                        trail.appendleft(middle[middle_semicolon_index:])
                         middle = rstripped + middle[trail_start:middle_semicolon_index]
                     else:
-                        trail = middle[trail_start:] + trail
+                        trail.appendleft(middle[trail_start:])
                         middle = rstripped
                     trimmed_something = True
 
+        trail = "".join(trail)
         return lead, middle, trail
 
     @staticmethod

django/utils/http.py

@@ -169,11 +169,11 @@ def int_to_base36(i):
         raise ValueError("Negative base36 conversion input.")
     if i < 36:
         return char_set[i]
-    b36 = ""
+    b36_parts = []
     while i != 0:
         i, n = divmod(i, 36)
-        b36 = char_set[n] + b36
-    return b36
+        b36_parts.append(char_set[n])
+    return "".join(reversed(b36_parts))
 
 
 def urlsafe_base64_encode(s):

django/utils/numberformat.py

@@ -91,15 +91,15 @@ def format(
             # grouping is a single value
             intervals = [grouping, 0]
         active_interval = intervals.pop(0)
-        int_part_gd = ""
+        int_part_gd = []
         cnt = 0
         for digit in int_part[::-1]:
             if cnt and cnt == active_interval:
                 if intervals:
                     active_interval = intervals.pop(0) or active_interval
-                int_part_gd += thousand_sep[::-1]
+                int_part_gd.append(thousand_sep[::-1])
                 cnt = 0
-            int_part_gd += digit
+            int_part_gd.append(digit)
             cnt += 1
-        int_part = int_part_gd[::-1]
+        int_part = "".join(int_part_gd)[::-1]
     return sign + int_part + dec_part

django/utils/text.py

@@ -103,7 +103,7 @@ class TruncationCompleted(Exception):
     def __init__(self, *, length, replacement, convert_charrefs=True):
         super().__init__(convert_charrefs=convert_charrefs)
         self.tags = deque()
-        self.output = ""
+        self.output = []
         self.remaining = length
         self.replacement = replacement
 
@@ -119,13 +119,13 @@ def handle_startendtag(self, tag, attrs):
             self.handle_endtag(tag)
 
     def handle_starttag(self, tag, attrs):
-        self.output += self.get_starttag_text()
+        self.output.append(self.get_starttag_text())
         if tag not in self.void_elements:
             self.tags.appendleft(tag)
 
     def handle_endtag(self, tag):
         if tag not in self.void_elements:
-            self.output += f"</{tag}>"
+            self.output.append(f"</{tag}>")
             try:
                 self.tags.remove(tag)
             except ValueError:
@@ -136,16 +136,16 @@ def handle_data(self, data):
         data_len = len(data)
         if self.remaining < data_len:
             self.remaining = 0
-            self.output += add_truncation_text(output, self.replacement)
+            self.output.append(add_truncation_text(output, self.replacement))
             raise self.TruncationCompleted
         self.remaining -= data_len
-        self.output += output
+        self.output.append(output)
 
     def feed(self, data):
         try:
             super().feed(data)
         except self.TruncationCompleted:
-            self.output += "".join([f"</{tag}>" for tag in self.tags])
+            self.output.extend([f"</{tag}>" for tag in self.tags])
             self.tags.clear()
             self.reset()
         else:
@@ -166,9 +166,9 @@ def __init__(self, *, length, replacement, convert_charrefs=True):
     def process(self, data):
         self.processed_chars += len(data)
         if (self.processed_chars == self.length) and (
-            len(self.output) + len(data) == len(self.rawdata)
+            sum(len(p) for p in self.output) + len(data) == len(self.rawdata)
         ):
-            self.output += data
+            self.output.append(data)
             raise self.TruncationCompleted
         output = escape("".join(data[: self.remaining]))
         return data, output
@@ -213,7 +213,7 @@ def chars(self, num, truncate=None, html=False):
             parser = TruncateCharsHTMLParser(length=length, replacement=truncate)
             parser.feed(text)
             parser.close()
-            return parser.output
+            return "".join(parser.output)
         return self._text_chars(length, truncate, text)
 
     def _text_chars(self, length, truncate, text):
@@ -250,7 +250,7 @@ def words(self, num, truncate=None, html=False):
             parser = TruncateWordsHTMLParser(length=length, replacement=truncate)
             parser.feed(self._wrapped)
             parser.close()
-            return parser.output
+            return "".join(parser.output)
         return self._text_words(length, truncate)
 
     def _text_words(self, length, truncate):

docs/topics/performance.txt

@@ -426,6 +426,12 @@ Django is compatible with versions of PyPy corresponding to the supported
 Python versions, but you will need to check the compatibility of other
 libraries you rely on.
 
+That said, a lot of a web framework's work is done by concatenating
+strings, and PyPy has an issue with that (see
+`this PyPy blog
+<https://pypy.org/posts/2023/01/string-concatenation-quadratic.html>`_).
+This may cause performance issues, depending on your use.
+
 C implementations of Python libraries
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/topics/signals.txt

@@ -323,11 +323,15 @@ in order to reduce the number of sync/async calling-style switches within a
 they are async before being called. This means that an asynchronous receiver
 registered before a synchronous receiver may be executed after the synchronous
 receiver. In addition, async receivers are executed concurrently using
-``asyncio.gather()``.
+:class:`asyncio.TaskGroup`.
 
 All built-in signals, except those in the async request-response cycle, are
 dispatched using :meth:`Signal.send`.
 
+.. versionchanged:: 6.1
+
+    In older versions, async receivers were executed via ``asyncio.gather()``.
+
 Disconnecting signals
 =====================
 

tests/backends/sqlite/tests.py

@@ -1,5 +1,6 @@
 import os
 import re
+import sqlite3
 import tempfile
 import threading
 import unittest
@@ -215,15 +216,28 @@ def test_parameter_quoting(self):
         substituted = "SELECT '\"''\\'"
         self.assertEqual(connection.queries[-1]["sql"], substituted)
 
-    def test_large_number_of_parameters(self):
-        # If SQLITE_MAX_VARIABLE_NUMBER (default = 999) has been changed to be
-        # greater than SQLITE_MAX_COLUMN (default = 2000), last_executed_query
-        # can hit the SQLITE_MAX_COLUMN limit (#26063).
-        with connection.cursor() as cursor:
-            sql = "SELECT MAX(%s)" % ", ".join(["%s"] * 2001)
-            params = list(range(2001))
-            # This should not raise an exception.
-            cursor.db.ops.last_executed_query(cursor.cursor, sql, params)
+    def test_parameter_count_exceeds_variable_or_column_limit(self):
+        sql = "SELECT MAX(%s)" % ", ".join(["%s"] * 1001)
+        params = list(range(1001))
+        for label, limit, current_limit in [
+            (
+                "variable",
+                sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER,
+                connection.features.max_query_params,
+            ),
+            (
+                "column",
+                sqlite3.SQLITE_LIMIT_COLUMN,
+                connection.connection.getlimit(sqlite3.SQLITE_LIMIT_COLUMN),
+            ),
+        ]:
+            with self.subTest(limit=label):
+                connection.connection.setlimit(limit, 1000)
+                self.addCleanup(connection.connection.setlimit, limit, current_limit)
+                with connection.cursor() as cursor:
+                    # This should not raise an exception.
+                    cursor.db.ops.last_executed_query(cursor.cursor, sql, params)
+                connection.connection.setlimit(limit, current_limit)
 
 
 @unittest.skipUnless(connection.vendor == "sqlite", "SQLite tests")