gh-69919: Catch all compile errors in the code module, pyrepl and IDLE (GH-157585)

compile() can raise MemoryError or RecursionError for too deeply
nested source, not only SyntaxError, OverflowError and ValueError.
IDLE's Shell then lost its prompt until the input was deleted.

Co-authored-by: Terry Jan Reedy <tjreedy@udel.edu>
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: 3 people

Doc/builtins/functions.rst

@@ -384,8 +384,14 @@ are always available.  They are listed here in alphabetical order.
    It is needed to unambiguous :ref:`filter <warning-filter>` syntax warnings
    by module name.
 
-   This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled
-   source is invalid.
+   This function raises :exc:`SyntaxError` if the compiled source is invalid,
+   including a *source* containing a null character or that cannot be decoded;
+   :exc:`ValueError` if *mode* or *flags* is invalid,
+   or if a string *source* contains surrogate characters;
+   :exc:`MemoryError` or :exc:`RecursionError` if *source* is too complex
+   to parse or compile,
+   for example an expression with many thousands of nested operators;
+   and :exc:`OverflowError` if *source* is too large.
 
    If you want to parse Python code into its AST representation, see
    :func:`ast.parse`.
@@ -417,11 +423,15 @@ are always available.  They are listed here in alphabetical order.
       Previously, :exc:`TypeError` was raised when null bytes were encountered
       in *source*.
 
-   .. versionadded:: 3.8
+   .. versionchanged:: 3.8
       ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
       support for top-level ``await``, ``async for``, and ``async with``.
 
-   .. versionadded:: 3.15
+   .. versionchanged:: 3.12
+      :exc:`SyntaxError` is raised instead of :exc:`ValueError` when null bytes
+      are encountered in *source*.
+
+   .. versionchanged:: 3.15
       Added the *module* parameter.
 
 

Doc/library/code.rst

@@ -92,7 +92,7 @@ Interactive Interpreter Objects
    *symbol* is ``'single'``.  One of several things can happen:
 
    * The input is incorrect; :func:`compile_command` raised an exception
-     (:exc:`SyntaxError` or :exc:`OverflowError`).  A syntax traceback will be
+     (usually :exc:`SyntaxError`).  A syntax traceback will be
      printed by calling the :meth:`showsyntaxerror` method.  :meth:`runsource`
      returns ``False``.
 

Lib/_pyrepl/console.py

@@ -257,7 +257,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
                 )
             self.showsyntaxerror(filename, source=source)
             return False
-        except (OverflowError, ValueError):
+        except Exception:
             self.showsyntaxerror(filename, source=source)
             return False
         if tree.body:
@@ -278,7 +278,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
                     )
                 self.showsyntaxerror(filename, source=source)
                 return False
-            except (OverflowError, ValueError):
+            except Exception:
                 self.showsyntaxerror(filename, source=source)
                 return False
 

Lib/_pyrepl/simple_interact.py

@@ -82,7 +82,7 @@ def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool:
     src = _strip_final_indent(unicodetext)
     try:
         code = console.compile(src, "<stdin>", "single")
-    except (OverflowError, SyntaxError, ValueError):
+    except Exception:
         lines = src.splitlines(keepends=True)
         if len(lines) == 1:
             return False

Lib/code.py

@@ -44,8 +44,8 @@ def runsource(self, source, filename="<input>", symbol="single"):
         One of several things can happen:
 
         1) The input is incorrect; compile_command() raised an
-        exception (SyntaxError or OverflowError).  A syntax traceback
-        will be printed by calling the showsyntaxerror() method.
+        exception (usually SyntaxError).  A syntax traceback will be
+        printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
@@ -62,7 +62,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
         """
         try:
             code = self.compile(source, filename, symbol)
-        except (OverflowError, SyntaxError, ValueError):
+        except Exception:
             # Case 1
             self.showsyntaxerror(filename, source=source)
             return False

Lib/idlelib/idle_test/test_runscript.py

@@ -29,6 +29,19 @@ def test_init(self):
         sb = runscript.ScriptBinding(ew)
         ew._close()
 
+    def test_checksyntax_compile_error(self):
+        # gh-69919: any error raised by compile() is reported.
+        ew = EditorWindow(root=self.root)
+        sb = runscript.ScriptBinding(ew)
+        sb.flist = mock.Mock()
+        sb.errorbox = mock.Mock()
+        with (mock.patch('idlelib.runscript.compile', create=True,
+                         side_effect=MemoryError()),
+              mock.patch('idlelib.runscript.open', mock.mock_open(read_data=b'x\n'))):
+            self.assertFalse(sb.checksyntax('test.py'))
+        sb.errorbox.assert_called_once_with('MemoryError', '<no detail available>')
+        ew._close()
+
     def test_run_module_event_shell_busy_no_restart(self):
         # gh-82183: running without restarting the busy shell aborts.
         ew = EditorWindow(root=self.root)

Lib/idlelib/pyshell.py

@@ -686,7 +686,7 @@ def execfile(self, filename, source=None):
                               + source + "\ndel __file__")
         try:
             code = compile(source, filename, "exec")
-        except (OverflowError, SyntaxError):
+        except Exception:
             self.tkconsole.resetoutput()
             print('*** Error in script or command!\n'
                  'Traceback (most recent call last):',
@@ -736,19 +736,23 @@ def showsyntaxerror(self, filename=None, **kwargs):
         text = tkconsole.text
         text.tag_remove("ERROR", "1.0", "end")
         type, value, tb = sys.exc_info()
-        msg = getattr(value, 'msg', '') or value or "<no detail available>"
-        lineno = getattr(value, 'lineno', '') or 1
-        offset = getattr(value, 'offset', '') or 0
+        if not issubclass(type, SyntaxError):
+            tkconsole.resetoutput()
+            InteractiveInterpreter.showsyntaxerror(self, filename, **kwargs)
+            tkconsole.showprompt()
+            return
+        msg = value.msg or "<no detail available>"
+        lineno = value.lineno or 1
+        offset = value.offset or 0
         if offset == 0:
             lineno += 1 #mark end of offending line
         if lineno == 1:
-            pos = "iomark + %d chars" % (offset-1)
+            pos = f"iomark + {offset-1} chars"
         else:
-            pos = "iomark linestart + %d lines + %d chars" % \
-                  (lineno-1, offset-1)
+            pos = f"iomark linestart + {lineno-1} lines + {offset-1} chars"
         tkconsole.colorize_syntax_error(text, pos)
         tkconsole.resetoutput()
-        self.write("SyntaxError: %s\n" % msg)
+        self.write(f"{type.__name__}: {msg}\n")
         tkconsole.showprompt()
 
     def showtraceback(self):

Lib/idlelib/runscript.py

@@ -93,15 +93,19 @@ def checksyntax(self, filename):
         try:
             # If successful, return the compiled code
             return compile(source, filename, "exec")
-        except (SyntaxError, OverflowError, ValueError) as value:
-            msg = getattr(value, 'msg', '') or value or "<no detail available>"
-            lineno = getattr(value, 'lineno', '') or 1
-            offset = getattr(value, 'offset', '') or 0
+        except SyntaxError as value:
+            msg = value.msg or "<no detail available>"
+            lineno = value.lineno or 1
+            offset = value.offset or 0
             if offset == 0:
                 lineno += 1  #mark end of offending line
             pos = "0.0 + %d lines + %d chars" % (lineno-1, offset-1)
             editwin.colorize_syntax_error(text, pos)
-            self.errorbox("SyntaxError", "%-20s" % msg)
+            self.errorbox(type(value).__name__, msg)
+            return False
+        except Exception as value:
+            msg = str(value) or "<no detail available>"
+            self.errorbox(type(value).__name__, msg)
             return False
         finally:
             shell.set_warning_stream(saved_stream)

Lib/pdb.py

@@ -156,7 +156,7 @@ def find_function(funcname, filename):
             if funcdef:
                 try:
                     code = compile(funcdef, filename, 'exec')
-                except SyntaxError:
+                except Exception:
                     continue
                 # We should always be able to find the code object here
                 funccode = next(c for c in code.co_consts if
@@ -2781,7 +2781,7 @@ def _compile_error_message(self, expr):
         """Return the error message as string if compiling `expr` fails."""
         try:
             compile(expr, "<stdin>", "eval")
-        except SyntaxError as exc:
+        except Exception as exc:
             return _rstr(self._format_exc(exc))
         return ""
 

Lib/test/test_code_module.py

@@ -140,6 +140,17 @@ def test_unicode_error(self):
         self.assertIsNone(self.sysmod.last_value.__traceback__)
         self.assertIs(self.sysmod.last_exc, self.sysmod.last_value)
 
+    def test_compile_error(self):
+        # Any error raised by compile() must be reported (gh-69919).
+        self.infunc.side_effect = ['-' * 100_000 + '1', EOFError('Finished')]
+        self.console.interact()
+        output = ''.join(''.join(call[1]) for call in self.stderr.method_calls)
+        output = output[output.index('(InteractiveConsole)'):]
+        output = output[output.index('\n') + 1:]
+        self.assertRegex(output, r'^(MemoryError|RecursionError): ')
+        self.assertIn(self.sysmod.last_type, (MemoryError, RecursionError))
+        self.assertIs(self.sysmod.last_exc, self.sysmod.last_value)
+
     def test_sysexcepthook(self):
         self.infunc.side_effect = ["def f():",
                                    "    raise ValueError('BOOM!')",