gh-155742: Use PyMem_Malloc() in decode_unicode_with_escapes() (#157587)

Replace soft deprecated PyBytes_FromStringAndSize(NULL, size) with a
simple PyMem_Malloc(). Avoid also "s = buf" to make the code easier to follow
(and easier to debug).

Add non-ASCII tests to test_string_literals.

Author: vstinner

Lib/test/test_string_literals.py

@@ -54,6 +54,10 @@
 assert ord(h) == 0x1d120
 i = r'\U0001d120'
 assert list(map(ord, i)) == [92, 85, 48, 48, 48, 49, 100, 49, 50, 48]
+j = 'ä'
+assert list(map(ord, j)) == [228]
+k = '\ä'
+assert list(map(ord, k)) == [92, 228]
 """
 
 
@@ -76,7 +80,7 @@ def test_template(self):
         # Check that the template doesn't contain any non-printables
         # except for \n.
         for c in TEMPLATE:
-            assert c == '\n' or ' ' <= c <= '~', repr(c)
+            assert c == '\n' or ' ' <= c <= '~' or c == 'ä', repr(c)
 
     def test_eval_str_normal(self):
         self.assertEqual(eval(""" 'x' """), 'x')
@@ -88,6 +92,21 @@ def test_eval_str_normal(self):
         self.assertEqual(eval(""" '\u1881' """), chr(0x1881))
         self.assertEqual(eval(r""" '\U0001d120' """), chr(0x1d120))
         self.assertEqual(eval(""" '\U0001d120' """), chr(0x1d120))
+        # Lone "\" character at the end
+        self.assertEqual(eval(r"'abc\\'"), 'abc\\')
+
+    def test_eval_str_unicode(self):
+        for s in (
+            'ϼўТλФЙ',
+            'A͏B ﬖ̳AA̝',
+            '\U00100000\U0010ffff\U0010fffd',
+            'ä',
+            '\\ä',
+            "\\П",
+            "áàäéèęöő.\\n",
+        ):
+            with self.subTest(s=s):
+                self.assertEqual(eval(f"{s!r}"), s)
 
     def test_eval_str_incomplete(self):
         self.assertRaises(SyntaxError, eval, r""" '\x' """)

Parser/string_parser.c

@@ -135,31 +135,27 @@ static PyObject *
 decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t)
 {
     PyObject *v;
-    PyObject *u;
-    char *buf;
     char *p;
     const char *end;
 
     /* check for integer overflow */
     if (len > (size_t)PY_SSIZE_T_MAX / 6) {
         return NULL;
     }
-    /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5
-       "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */
-    u = PyBytes_FromStringAndSize((char *)NULL, (Py_ssize_t)len * 6);
-    if (u == NULL) {
-        return NULL;
-    }
-    p = buf = PyBytes_AsString(u);
-    if (p == NULL) {
+    /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5.
+     * "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6. */
+    Py_ssize_t alloc = (Py_ssize_t)len * 6;
+    char *buf = PyMem_Malloc(alloc);
+    if (buf == NULL) {
         return NULL;
     }
+    p = buf;
     end = s + len;
     while (s < end) {
         if (*s == '\\') {
             *p++ = *s++;
             if (s >= end || *s & 0x80) {
-                strcpy(p, "u005c");
+                memcpy(p, "u005c", 5);
                 p += 5;
                 if (s >= end) {
                     break;
@@ -174,46 +170,46 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t)
             Py_ssize_t i;
             w = decode_utf8(&s, end);
             if (w == NULL) {
-                Py_DECREF(u);
+                PyMem_Free(buf);
                 return NULL;
             }
             kind = PyUnicode_KIND(w);
             data = PyUnicode_DATA(w);
             w_len = PyUnicode_GET_LENGTH(w);
             for (i = 0; i < w_len; i++) {
+                // sprintf() writes a null byte: the buffer is large enough
+                // for that thanks to the overallocation.
+                assert((p + 11 - buf) <= alloc);
                 Py_UCS4 chr = PyUnicode_READ(kind, data, i);
                 sprintf(p, "\\U%08x", chr);
                 p += 10;
             }
-            /* Should be impossible to overflow */
-            assert(p - buf <= PyBytes_GET_SIZE(u));
             Py_DECREF(w);
         }
         else {
             *p++ = *s++;
         }
     }
     len = (size_t)(p - buf);
-    s = buf;
 
     int first_invalid_escape_char;
     const char *first_invalid_escape_ptr;
-    v = _PyUnicode_DecodeUnicodeEscapeInternal2(s, (Py_ssize_t)len, NULL, NULL,
+    v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL, NULL,
                                                 &first_invalid_escape_char,
                                                 &first_invalid_escape_ptr);
 
     // HACK: later we can simply pass the line no, since we don't preserve the tokens
     // when we are decoding the string but we preserve the line numbers.
     if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL) {
-        if (warn_invalid_escape_sequence(parser, s, first_invalid_escape_ptr, t) < 0) {
-            /* We have not decref u before because first_invalid_escape_ptr
-               points inside u. */
-            Py_XDECREF(u);
+        if (warn_invalid_escape_sequence(parser, buf, first_invalid_escape_ptr, t) < 0) {
+            /* We have not deallocated the buffer before because
+             * first_invalid_escape_ptr points inside buf. */
+            PyMem_Free(buf);
             Py_DECREF(v);
             return NULL;
         }
     }
-    Py_XDECREF(u);
+    PyMem_Free(buf);
     return v;
 }