Merge branch 'main' into dtoa-precision-limit

Author: gpshead

Doc/c-api/unicode.rst

@@ -1945,7 +1945,7 @@ object.
    On success, return ``0``.
    On error, set an exception, leave the writer unchanged, and return ``-1``.
 
-.. c:function:: int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, const char *string, Py_ssize_t length, const char *errors, Py_ssize_t *consumed)
+.. c:function:: int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, const char *str, Py_ssize_t size, const char *errors, Py_ssize_t *consumed)
 
    Decode the string *str* from UTF-8 with *errors* error handler and write the
    output into *writer*.

Include/cpython/unicodeobject.h

@@ -516,8 +516,8 @@ PyAPI_FUNC(int) PyUnicodeWriter_Format(
     ...);
 PyAPI_FUNC(int) PyUnicodeWriter_DecodeUTF8Stateful(
     PyUnicodeWriter *writer,
-    const char *string,         /* UTF-8 encoded string */
-    Py_ssize_t length,          /* size of string */
+    const char *str,            /* UTF-8 encoded string */
+    Py_ssize_t size,            /* size of string */
     const char *errors,         /* error handling */
     Py_ssize_t *consumed);      /* bytes consumed */
 
@@ -538,6 +538,9 @@ typedef struct {
     /* minimum character (default: 127, ASCII) */
     Py_UCS4 min_char;
 
+    // If non-zero, _PyUnicodeWriter_Finish() needs to check maxchar.
+    int recheck_maxchar;
+
     /* If non-zero, overallocate the buffer (default: 0). */
     unsigned char overallocate;
 
@@ -554,25 +557,33 @@ typedef struct {
 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(void) _PyUnicodeWriter_Init(
     _PyUnicodeWriter *writer);
 
-/* Prepare the buffer to write 'length' characters
-   with the specified maximum character.
-
-   Return 0 on success, raise an exception and return -1 on error. */
-#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR)             \
-    (((MAXCHAR) <= (WRITER)->maxchar                                  \
-      && (LENGTH) <= (WRITER)->size - (WRITER)->pos)                  \
-     ? 0                                                              \
-     : (((LENGTH) == 0)                                               \
-        ? 0                                                           \
-        : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR))))
-
-/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro
-   instead. */
+// Don't call this function directly, use _PyUnicodeWriter_Prepare() instead.
 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_PrepareInternal(
     _PyUnicodeWriter *writer,
     Py_ssize_t length,
     Py_UCS4 maxchar);
 
+// Prepare the buffer to write 'length' characters
+// with the specified maximum character.
+//
+// Return 0 on success. Set an exception and return -1 on error.
+_Py_DEPRECATED_EXTERNALLY(3.14) static inline int
+_PyUnicodeWriter_Prepare(_PyUnicodeWriter *writer,
+                         Py_ssize_t length, Py_UCS4 maxchar)
+{
+    assert(0 <= length);
+    if (maxchar <= writer->maxchar && length <= (writer->size - writer->pos)) {
+        return 0;
+    }
+    if (length == 0) {
+        return 0;
+    }
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
+    return _PyUnicodeWriter_PrepareInternal(writer, length, maxchar);
+_Py_COMP_DIAG_POP
+}
+
 /* Prepare the buffer to have at least the kind KIND.
    For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will
    support characters in range U+000-U+FFFF.

Include/internal/pycore_unicodeobject.h

@@ -130,38 +130,51 @@ _PyUnicodeWriter_CanWrite(_PyUnicodeWriter *writer)
 #endif
 
 static inline void
-_PyUnicodeWriter_Update(_PyUnicodeWriter *writer)
+_PyUnicodeWriter_SetBuffer(_PyUnicodeWriter *writer, PyObject *buffer)
 {
-    PyObject *buffer = writer->buffer;
-    writer->maxchar = PyUnicode_MAX_CHAR_VALUE(buffer);
+    assert(writer->pos <= PyUnicode_GET_LENGTH(buffer));
+
+    // Py_DECREF() the previous buffer (if any)
+    Py_XSETREF(writer->buffer, buffer);
     writer->data = PyUnicode_DATA(buffer);
     writer->kind = PyUnicode_KIND(buffer);
+    writer->maxchar = PyUnicode_MAX_CHAR_VALUE(buffer);
+    writer->size = PyUnicode_GET_LENGTH(buffer);
+    writer->readonly = 0;
+}
 
-    if (!writer->readonly) {
-        writer->size = PyUnicode_GET_LENGTH(buffer);
-    }
-    else {
-        /* Copy-on-write mode: set buffer size to 0 so
-         * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on
-         * next write. */
-        writer->size = 0;
-    }
+static inline void
+_PyUnicodeWriter_SetReadOnly(_PyUnicodeWriter *writer, PyObject *obj,
+                             Py_ssize_t length)
+{
+    assert(writer->buffer == NULL);
+    assert(writer->pos == 0);
+    // Micro-optimization: pass length as a parameter, as it's usually known
+    // by the caller
+    assert(length == PyUnicode_GET_LENGTH(obj));
+
+    writer->buffer = obj;
+    writer->data = NULL;
+    /* Set kind and size to 0 to make sure that the next
+     * _PyUnicodeWriter_Prepare() call allocates a new buffer and copies
+     * characters. */
+    writer->kind = 0;
+    writer->maxchar = PyUnicode_MAX_CHAR_VALUE(obj);
+    writer->size = 0;
+    writer->pos = length;
+    writer->readonly = 1;
 }
 
 static inline int
 _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch)
 {
-    if (ch > writer->maxchar || 1 > writer->size - writer->pos) {
+    if (ch > writer->maxchar || 1 > (writer->size - writer->pos)) {
         if (writer->buffer == NULL && ch <= 255) {
             // If the first write is a Latin1 character, use the singleton
             // as a read-only object
             PyObject *obj = _Py_LATIN1_CHR(ch);
-            writer->readonly = 1;
-            writer->buffer = obj; // Py_NewRef() is not need on immortal object
-            _PyUnicodeWriter_Update(writer);
-            assert(writer->pos == 0);
-            writer->pos = 1;
-            // The next write will create a new buffer and copy the string
+            // Py_NewRef() is not needed on immortal object
+            _PyUnicodeWriter_SetReadOnly(writer, obj, 1);
             return 0;
         }
 
@@ -176,6 +189,10 @@ _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch)
     return 0;
 }
 
+extern PyObject* _PyUnicodeWriter_FinishWithSize(
+    _PyUnicodeWriter *writer,
+    Py_ssize_t size);
+
 /* --- Unicode API -------------------------------------------------------- */
 
 // Export for '_json' shared extension

Lib/test/test_capi/test_unicode.py

@@ -461,6 +461,14 @@ def check_format(expected, format, *args):
         check_format('%abc',
                      b'%%%s', b'abc')
 
+        # test "%s" with empty string
+        check_format('x=',
+                     b'x=%s', b'')
+        check_format('x=',
+                     b'x=%0s', b'')
+        check_format('x=',
+                     b'x=%.3s', b'')
+
         # truncated string
         check_format('abc',
                      b'%.3s', b'abcdef')
@@ -1970,13 +1978,35 @@ def test_ascii(self):
         writer.write_ascii(b"Python! <truncated>", 6)
         self.assertEqual(writer.finish(), "Hello Python")
 
+    def test_write_latin1(self):
+        # Test _PyUnicodeWriter_WriteLatin1String()
+        writer = self.create_writer(0)
+        # Start with ASCII buffer
+        writer.write_latin1(b"abc IGNORED", 3)
+        writer.write_latin1(b"IGNORED", 0)
+        # Change buffer kind to UCS-1
+        writer.write_latin1(b"\xe9", 1)
+        # Change buffer kind to UCS-2
+        writer.write_str('[\u20ac]')
+        writer.write_latin1(b"def\xa0", 4)
+        # Change buffer kind to UCS-4
+        writer.write_str('[\U0010ffff]')
+        writer.write_latin1(b"ghi\xff.", 5)
+        writer.write_latin1(b"IGNORED", 0)
+        self.assertEqual(writer.finish(),
+                         "abc\xe9[\u20ac]def\xa0[\U0010ffff]ghi\xff.")
+
     def test_invalid_utf8(self):
         writer = self.create_writer(0)
         with self.assertRaises(UnicodeDecodeError):
             writer.write_utf8(b"invalid=\xFF", -1)
 
     def test_recover_utf8_error(self):
-        # test recovering from PyUnicodeWriter_WriteUTF8() error
+        # Recover from PyUnicodeWriter_WriteUTF8() errors. A temporary write
+        # changes the buffer kind to UCS-2 before raising UnicodeDecodeError.
+        # Then, PyUnicodeWriter_Finish() has to change the buffer kind back to
+        # ASCII.
+
         writer = self.create_writer(0)
         writer.write_utf8(b"value=", -1)
 
@@ -1986,12 +2016,13 @@ def test_recover_utf8_error(self):
         with self.assertRaises(UnicodeDecodeError):
             s = "truncated\u20AC".encode()
             writer.write_utf8(s, len(s) - 1)
+        with self.assertRaises(UnicodeDecodeError):
+            # Change buffer kind to UCS-2 then raise UnicodeDecodeError
+            s = "\u20AC\u20AC".encode()
+            writer.write_utf8(s, len(s) - 1)
 
-        # retry write with a valid string
         writer.write_utf8(b"valid", -1)
-
-        self.assertEqual(writer.finish(),
-                         "value=valid")
+        self.assertEqual(writer.finish(), "value=valid")
 
     def test_decode_utf8(self):
         # test PyUnicodeWriter_DecodeUTF8Stateful()
@@ -2262,6 +2293,35 @@ def __repr__(self):
                 self.assertEqual(writer.get_buffer(), expected)
                 self.assertIs(writer.finish(), unique_string)
 
+    def test_readonly_optim_large_int(self):
+        # Read-only optimization in _PyLong_FormatWriter() for large integer:
+        # use _pylong.int_to_decimal_string() result as a read-only string.
+        # See pylong_int_to_decimal_string().
+
+        self.addCleanup(sys.set_int_max_str_digits,
+                        sys.get_int_max_str_digits())
+        sys.set_int_max_str_digits(0)
+
+        # _PyLong_FormatWriter() calls _pylong.int_to_decimal_string() for
+        # integer with Py_SIZE() > 1000.
+        large_int = 1 << (sys.int_info.bits_per_digit * 1020)
+        large_int_str = str(large_int)
+        expected = (len(large_int_str), 127, True)
+
+        for size in (0, 123):
+            with self.subTest(size=size):
+                # Test PyUnicodeWriter_WriteStr()
+                writer = self.create_writer(size)
+                writer.write_str(large_int)
+                self.assertEqual(writer.get_buffer(), expected)
+                self.assertEqual(writer.finish(), large_int_str)
+
+                # Test PyUnicodeWriter_WriteRepr()
+                writer = self.create_writer(size)
+                writer.write_repr(large_int)
+                self.assertEqual(writer.get_buffer(), expected)
+                self.assertEqual(writer.finish(), large_int_str)
+
 
 # Test PyUnicodeWriter_Format()
 @unittest.skipIf(ctypes is None, 'need ctypes')
@@ -2299,6 +2359,29 @@ def test_recover_error(self):
 
         self.assertEqual(writer.finish(), 'Hello World.')
 
+    def test_recheck_maxchar(self):
+        # PyUnicodeWriter_Format() changes buffer kind to UCS-2 before raising
+        # an exception. Then, PyUnicodeWriter_Finish() has to change the buffer
+        # kind back to ASCII.
+        from ctypes import py_object
+
+        class StrError:
+            def __str__(self):
+                raise RuntimeError("bug")
+
+        writer = self.create_writer(0)
+        # Allocate ASCII buffer
+        writer.write_str('ascii')
+
+        obj = StrError()
+        ucs2_utf8 = '\u20ac'.encode()
+        with self.assertRaises(RuntimeError):
+            # Change buffer kind to UCS-2, but then raise RuntimeError
+            self.writer_format(writer, b"%s%S", ucs2_utf8, py_object(obj))
+
+        writer.write_str('.')
+        self.assertEqual(writer.finish(), 'ascii.')
+
     def test_readonly_optim(self):
         # Read-only optimization: if the first and only write is a Python str
         # object and no buffer was allocated yet, return the object unchanged

Lib/test/test_format.py

@@ -240,6 +240,8 @@ def test_common_format(self):
         testcommon("%d", 42, "42")
         testcommon("%d", -42, "-42")
         testcommon("%d", 42.0, "42")
+        testcommon("%#d", 42, "42")
+        testcommon("%#d", -42, "-42")
         testcommon("%#x", 1, "0x1")
         testcommon("%#X", 1, "0X1")
         testcommon("%#o", 1, "0o1")
@@ -250,8 +252,12 @@ def test_common_format(self):
         testcommon("%#X", 0, "0X0")
         testcommon("%x", 0x42, "42")
         testcommon("%x", -0x42, "-42")
+        testcommon("%#x", 0x42, "0x42")
+        testcommon("%#x", -0x42, "-0x42")
         testcommon("%o", 0o42, "42")
         testcommon("%o", -0o42, "-42")
+        testcommon("%#o", 0o42, "0o42")
+        testcommon("%#o", -0o42, "-0o42")
         # alternate float formatting
         testcommon('%g', 1.1, '1.1')
         testcommon('%#g', 1.1, '1.10000')
@@ -344,12 +350,21 @@ def test_common_format(self):
                         "format argument 1: %g requires a real number, not str")
 
     def test_str_format(self):
+        testformat("%s", "abc", "abc")
         testformat("%r", "\u0378", "'\\u0378'")  # non printable
         testformat("%a", "\u0378", "'\\u0378'")  # non printable
         testformat("%r", "\u0374", "'\u0374'")   # printable
         testformat("%a", "\u0374", "'\\u0374'")  # printable
         testformat('%(x)r', {'x': 1}, '1')
 
+        # Some small ints
+        for fmt in ('s', 'r', 'a'):
+            with self.subTest(fmt=fmt):
+                testformat("%" + fmt, 42, "42")
+                testformat("%#" + fmt, 42, "42")
+                testformat("%" + fmt, -42, "-42")
+                testformat("%#" + fmt, -42, "-42")
+
         # Test exception for unknown format characters, etc.
         if verbose:
             print('Testing exceptions')