[3.12] gh-158446: Reject float format precision near INT_MAX (GH-158474) (#158479)

* gh-158446: Reject float format precision near INT_MAX (GH-158474)

Formatting a float or complex with a precision within about 1000 of
INT_MAX could crash or produce incorrect output.
PyOS_double_to_string() now raises ValueError("precision too big") for
such precisions, as the format string parsers already do for
precisions above INT_MAX.

The limit applies regardless of presentation type or value, so a few
calls that previously succeeded (inf, nan, or 'g' with such a
precision) now raise as well.
(cherry picked from commit b7b4f3e)

Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com>

* gh-158446: Fix the new test_format test on 3.12

test_format.py does not import import_module on this branch.  Import
INT_MAX from _testcapi directly, as the neighboring test does.

---------

Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com>
Co-authored-by: Gregory P. Smith <greg@krypto.org>

Author: miss-islington

Lib/test/test_format.py

@@ -488,6 +488,27 @@ def test_precision_c_limits(self):
         with self.assertRaises(ValueError) as cm:
             format(c, ".%sf" % (INT_MAX + 1))
 
+    @support.cpython_only
+    def test_precision_near_int_max(self):
+        # gh-158446: Precisions just below INT_MAX are rejected before any
+        # output buffer size is computed from them.
+        from _testcapi import INT_MAX
+
+        f = 1e300
+        c = complex(f)
+        for prec in (INT_MAX, INT_MAX - 1023):
+            for code in "feg":
+                spec = ".%d%s" % (prec, code)
+                with self.subTest(spec=spec):
+                    with self.assertRaises(ValueError):
+                        format(f, spec)
+                    with self.assertRaises(ValueError):
+                        format(c, spec)
+                    with self.assertRaises(ValueError):
+                        ("%" + spec) % f
+                    with self.assertRaises(ValueError):
+                        ("%" + spec).encode() % f
+
     def test_g_format_has_no_trailing_zeros(self):
         # regression test for bugs.python.org/issue40780
         self.assertEqual("%.3g" % 1505.0, "1.5e+03")

Misc/NEWS.d/next/Security/2026-09-29-16-32-46.gh-issue-158446.dToaPr.rst

@@ -0,0 +1,5 @@
+Fix a crash or incorrect output that could occur when formatting a
+:class:`float` or :class:`complex` with a precision close to the platform's
+``INT_MAX``.  :c:func:`PyOS_double_to_string` now raises :exc:`ValueError` for
+any precision of that magnitude, regardless of presentation type or value, as
+the format string parsers already did for precisions above ``INT_MAX``.

Python/dtoa.c

@@ -67,6 +67,10 @@
  *  8. A corner case where _Py_dg_dtoa didn't strip trailing zeros has been
  *     fixed. (bugs.python.org/issue40780)
  *
+ *  9. _Py_dg_dtoa clamps ndigits in modes 3 and 5 so that its buffer size
+ *     arithmetic cannot exceed the int range, and rv_alloc's size doubling
+ *     uses size_t. (gh-158446)
+ *
  ***************************************************************/
 
 /* Please send bug reports for the original dtoa.c code to David M. Gay (dmg
@@ -2123,7 +2127,8 @@ _Py_dg_strtod(const char *s00, char **se)
 static char *
 rv_alloc(int i)
 {
-    int j, k, *r;
+    int k, *r;
+    size_t j;   /* size_t so that j <<= 1 cannot overflow for i near INT_MAX */
 
     j = sizeof(ULong);
     for(k = 0;
@@ -2387,6 +2392,15 @@ _Py_dg_dtoa(double dd, int mode, int ndigits,
         leftright = 0;
         /* fall through */
     case 5:
+        /* -330 < k < 330 for any finite nonzero double.  Clamp ndigits so
+           that ndigits + k + 1 stays within int range; no double has
+           anywhere near this many decimal digits so the digits returned
+           are unaffected (*decpt saturates in the no_digits case).  Same
+           bound as DOUBLE_TO_STRING_PRECISION_MAX in pystrtod.c. */
+        if (ndigits > INT_MAX - 1024)
+            ndigits = INT_MAX - 1024;
+        else if (ndigits < -(INT_MAX - 1024))
+            ndigits = -(INT_MAX - 1024);
         i = ndigits + k + 1;
         ilim = i;
         ilim1 = i - 1;

Python/pystrtod.c

@@ -400,6 +400,15 @@ _Py_string_to_number_with_underscores(
     return NULL;
 }
 
+/* Largest precision magnitude accepted by PyOS_double_to_string().  The
+   output buffer sizes computed below and within _Py_dg_dtoa() use int and
+   Py_ssize_t arithmetic on roughly precision + (digits before the point, at
+   most DBL_MAX_10_EXP + 1 == 309) + a few bytes of sign, point and exponent.
+   Staying this far inside the int range keeps all of those sums in range.
+   (Only C callers can pass a negative precision.)  _Py_dg_dtoa() applies
+   the same bound to its ndigits argument. */
+#define DOUBLE_TO_STRING_PRECISION_MAX (INT_MAX - 1024)
+
 #if _PY_SHORT_FLOAT_REPR == 0
 
 /* Given a string that may have a decimal point in the current
@@ -765,6 +774,13 @@ char * PyOS_double_to_string(double val,
     int t, exp;
     int upper = 0;
 
+    if (precision > DOUBLE_TO_STRING_PRECISION_MAX
+        || precision < -DOUBLE_TO_STRING_PRECISION_MAX)
+    {
+        PyErr_SetString(PyExc_ValueError, "precision too big");
+        return NULL;
+    }
+
     /* Validate format_code, and map upper and lower case */
     switch (format_code) {
     case 'e':          /* exponent */
@@ -1226,6 +1242,13 @@ char * PyOS_double_to_string(double val,
     const char * const *float_strings = lc_float_strings;
     int mode;
 
+    if (precision > DOUBLE_TO_STRING_PRECISION_MAX
+        || precision < -DOUBLE_TO_STRING_PRECISION_MAX)
+    {
+        PyErr_SetString(PyExc_ValueError, "precision too big");
+        return NULL;
+    }
+
     /* Validate format_code, and map upper and lower case. Compute the
        mode and make any adjustments as needed. */
     switch (format_code) {