[3.15] gh-69371: Fix pydoc for modules whose path contains undecodable bytes (GH-155239) (GH-156608)

"python -m pydoc -w" and the pydoc HTTP server failed with
UnicodeEncodeError.  The file URL is now created with
urllib.request.pathname2url(), which percent-encodes the path using the
filesystem encoding, and characters unencodable in the generated UTF-8
page are escaped with backslashes.
(cherry picked from commit c8ca336)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: 3 people

Lib/pydoc.py

@@ -71,7 +71,6 @@ class or function within a module or module in a package.  If the
 import textwrap
 import time
 import tokenize
-import urllib.parse
 import warnings
 from annotationlib import Format
 from collections import deque
@@ -806,7 +805,8 @@ def docmodule(self, object, name=None, mod=None, *ignored):
         head = '<strong class="title">%s</strong>' % linkedname
         try:
             path = inspect.getabsfile(object)
-            url = urllib.parse.quote(path)
+            import urllib.request
+            url = urllib.request.pathname2url(path)
             filelink = self.filelink(url, path)
         except TypeError:
             filelink = '(built-in)'
@@ -1765,7 +1765,8 @@ def writedoc(thing, forceload=0):
     """Write HTML documentation to a file in the current directory."""
     object, name = resolve(thing, forceload)
     page = html.page(describe(object), html.document(object, name))
-    with open(name + '.html', 'w', encoding='utf-8') as file:
+    with open(name + '.html', 'w', encoding='utf-8',
+              errors='backslashreplace') as file:
         file.write(page)
     print('wrote', name + '.html')
 
@@ -2362,7 +2363,7 @@ def do_GET(self):
             self.send_header('Content-Type', '%s; charset=UTF-8' % content_type)
             self.end_headers()
             self.wfile.write(self.urlhandler(
-                self.path, content_type).encode('utf-8'))
+                self.path, content_type).encode('utf-8', 'backslashreplace'))
 
         def log_message(self, *args):
             # Don't log messages.

Lib/test/test_pydoc/test_pydoc.py

@@ -19,6 +19,7 @@
 import unittest
 import unittest.mock
 import urllib.parse
+import urllib.request
 import xml.etree
 import xml.etree.ElementTree
 import textwrap
@@ -429,7 +430,7 @@ def test_html_doc(self):
         expected_lines = [line.strip() for line in expected_lines if line]
         self.assertEqual(text_lines, expected_lines)
         mod_file = inspect.getabsfile(pydoc_mod)
-        mod_url = urllib.parse.quote(mod_file)
+        mod_url = urllib.request.pathname2url(mod_file)
         self.assertIn(mod_url, result)
         self.assertIn(mod_file, result)
         self.assertIn(doc_loc, result)
@@ -1024,6 +1025,31 @@ def test_synopsis_sourceless_empty_doc(self):
             synopsis_cached = pydoc.synopsis(cached_path, {})
             self.assertIsNone(synopsis_cached)
 
+    @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE,
+                         'requires undecodable file names')
+    def test_html_doc_undecodable_path(self):
+        # gh-69371: the path of the module is not encodable in UTF-8.
+        with os_helper.temp_cwd() as test_dir:
+            subdir = os.path.join(os.fsencode(test_dir),
+                                  os_helper.TESTFN_UNDECODABLE)
+            try:
+                os.mkdir(subdir)
+            except OSError:
+                self.skipTest('undecodable paths are not supported')
+            with open(os.path.join(subdir, b'undecodable_mod.py'), 'w') as f:
+                f.write('"""Module docstring."""\n')
+            with import_helper.DirsOnSysPath(os.fsdecode(subdir)):
+                mod = import_helper.import_fresh_module('undecodable_mod')
+                doc = pydoc.HTMLDoc().docmodule(mod)
+                with captured_stdout():
+                    pydoc.writedoc(mod)
+            # The link contains the percent-encoded path...
+            path = inspect.getabsfile(mod)
+            self.assertIn(urllib.request.pathname2url(path), doc)
+            # ...and the page can be written and served as UTF-8.
+            with open('undecodable_mod.html', encoding='utf-8') as f:
+                self.assertIn('undecodable_mod', f.read())
+
     def test_splitdoc_with_description(self):
         example_string = "I Am A Doc\n\n\nHere is my description"
         self.assertEqual(pydoc.splitdoc(example_string),

Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst

@@ -0,0 +1,5 @@
+Fix :mod:`pydoc` for modules whose path contains undecodable bytes.
+:func:`!pydoc.writedoc` and the pydoc HTTP server no longer fail with
+:exc:`UnicodeEncodeError`: the file URL is now percent-encoded using the
+filesystem encoding, and characters unencodable in the generated HTML page
+are escaped with backslashes.