Merge remote-tracking branch 'upstream/3.10' into security-310-wrap-bio

Author: pablogsal

Doc/library/ssl.rst

@@ -1750,6 +1750,12 @@ to speed up repeated connections from the same clients.
    :class:`SSLContext` representing a certificate chain that matches the server
    name.
 
+   If the callback assigns a new context to :attr:`SSLSocket.context`, any
+   further ClientHello message on the same connection (for example after a
+   TLS 1.3 HelloRetryRequest) is dispatched to the new context's
+   *sni_callback*, if it has one; the original callback is not called again
+   for that connection.
+
    Due to the early negotiation phase of the TLS connection, only limited
    methods and attributes are usable like
    :meth:`SSLSocket.selected_alpn_protocol` and :attr:`SSLSocket.context`.
@@ -1773,6 +1779,11 @@ to speed up repeated connections from the same clients.
 
    .. versionadded:: 3.7
 
+   .. versionchanged:: next
+      After the callback assigns a new :attr:`SSLSocket.context`, later
+      ClientHello messages on the connection are dispatched to the new
+      context's *sni_callback*.
+
 .. attribute:: SSLContext.set_servername_callback(server_name_callback)
 
    This is a legacy API retained for backwards compatibility. When possible,

Doc/library/urllib.request.rst

@@ -922,8 +922,14 @@ These methods are available on :class:`HTTPPasswordMgr` and
 
    *uri* can be either a single URI, or a sequence of URIs. *realm*, *user* and
    *passwd* must be strings. This causes ``(user, passwd)`` to be used as
-   authentication tokens when authentication for *realm* and a super-URI of any of
-   the given URIs is given.
+   authentication tokens when authentication for *realm* and a super-URI of any
+   of the given URIs is given. If a URI includes a scheme, its credentials only
+   match authentication URIs with the same scheme or no scheme. A URI without a
+   scheme matches authentication URIs with any scheme.
+
+   .. versionchanged:: next
+      Authentication credentials for URIs with a scheme are now scoped by
+      that scheme.
 
 
 .. method:: HTTPPasswordMgr.find_user_password(realm, authuri)

Lib/configparser.py

@@ -570,7 +570,8 @@ class RawConfigParser(MutableMapping):
     _OPT_TMPL = r"""
         (?P<option>                        # very permissive!
             (?:(?!{delim})\S)*             # non-delimiter non-whitespace
-            (?:\s+(?:(?!{delim})\S)+)*)    # optionally more words
+            (?:(?:(?!{delim})\s)+          # optionally more
+             (?:(?!{delim})\S)+)*)         #   space-separated words
         \s*(?P<vi>{delim})\s*              # any number of space/tab,
                                            # followed by any of the
                                            # allowed delimiters,
@@ -580,7 +581,8 @@ class RawConfigParser(MutableMapping):
     _OPT_NV_TMPL = r"""
         (?P<option>                        # very permissive!
             (?:(?!{delim})\S)*             # non-delimiter non-whitespace
-            (?:\s+(?:(?!{delim})\S)+)*)    # optionally more words
+            (?:(?:(?!{delim})\s)+          # optionally more
+             (?:(?!{delim})\S)+)*)         #   space-separated words
         \s*(?:                             # any number of space/tab,
         (?P<vi>{delim})\s*                 # optionally followed by
                                            # any of the allowed

Lib/stringprep.py

@@ -2674,9 +2674,11 @@ def makelink_with_filter(self, tarinfo, targetpath,
                     "makelink_with_filter: if filter_function is not None, "
                     + "extraction_root must also not be None")
             try:
-                filter_function(
+                filtered = filter_function(
                     unfiltered.replace(name=tarinfo.name, deep=False),
                     extraction_root)
+                if filtered is None:
+                    return
                 filtered = filter_function(unfiltered, extraction_root)
             except _FILTER_ERRORS as cause:
                 raise LinkFallbackError(tarinfo, unfiltered.name) from cause

Lib/tarfile.py

@@ -1547,6 +1547,15 @@ def test_builtin_encode(self):
         self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org")
         self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.")
 
+    @support.subTests(['unicode', 'encoded'], [
+        ('\N{CHEROKEE LETTER A}\N{CHEROKEE LETTER A}', b"xn--58da"),
+        ('\N{GEORGIAN CAPITAL LETTER AN}.', b"xn--7md."),
+        ('\N{CYRILLIC LETTER PALOCHKA}.example', b"xn--d5a.example"),
+        ('\N{ROMAN NUMERAL REVERSED ONE HUNDRED}.example.', b"xn--q5g.example."),
+    ])
+    def test_new_unicode_case_folding(self, unicode, encoded):
+        self.assertEqual(unicode.encode("idna"), encoded)
+
     def test_builtin_decode_length_limit(self):
         with self.assertRaisesRegex(UnicodeError, "too long"):
             (b"xn--016c"+b"a"*1100).decode("idna")

Lib/test/test_codecs.py

@@ -45,7 +45,7 @@ class CfgParserTestCaseClass:
     default_section = configparser.DEFAULTSECT
     interpolation = configparser._UNSET
 
-    def newconfig(self, defaults=None):
+    def newconfig(self, defaults=None, **kwargs):
         arguments = dict(
             defaults=defaults,
             allow_no_value=self.allow_no_value,
@@ -58,6 +58,7 @@ def newconfig(self, defaults=None):
             default_section=self.default_section,
             interpolation=self.interpolation,
         )
+        arguments.update(kwargs)
         instance = self.config_class(**arguments)
         return instance
 
@@ -360,6 +361,32 @@ def test_basic(self):
                 the larch {0[1]} 1
             """.format(self.delimiters)))
 
+    @support.subTests('data', [
+        'foo bar=baz',
+        'foo   bar=baz',
+        'foo=bar=baz',
+        'foo = bar=baz',
+        'foo\t \t=\t \tbar=baz',
+    ])
+    def test_space_delimiter(self, data):
+        # gh-156353: Space should be accepted as a delimiter
+        cf = self.newconfig(delimiters=(' ', '='))
+        cf.read_string(f"[all]\n{data}")
+        self.assertEqual(cf.options('all'), ['foo'])
+        self.assertEqual(cf.get('all', 'foo'), 'bar=baz')
+
+    @support.subTests('delimiter', ' =:;#x\t\0\N{RS}\N{CEDILLA}\N{CAT}')
+    @support.subTests('space_before', ['', ' ', '\t', ' \t'])
+    @support.subTests('space_after', ['', ' ', '\t', ' \t'])
+    def test_any_delimiter(self, delimiter, space_before, space_after):
+        cf = self.newconfig(
+            delimiters=(delimiter,),
+            inline_comment_prefixes=None,
+        )
+        cf.read_string(f"[all]\nfoo{space_before}{delimiter}{space_after}bar=baz")
+        self.assertEqual(cf.options('all'), ['foo'])
+        self.assertEqual(cf.get('all', 'foo'), 'bar=baz')
+
     def test_basic_from_dict(self):
         config = {
             "Foo Bar": {
@@ -1956,8 +1983,8 @@ class ConvertersTestCase(BasicTestCase, unittest.TestCase):
 
     config_class = configparser.ConfigParser
 
-    def newconfig(self, defaults=None):
-        instance = super().newconfig(defaults=defaults)
+    def newconfig(self, defaults=None, **kwargs):
+        instance = super().newconfig(defaults=defaults, **kwargs)
         instance.converters['list'] = lambda v: [e.strip() for e in v.split()
                                                  if e.strip()]
         return instance

Lib/test/test_configparser.py

@@ -488,6 +488,27 @@ def test_precision_c_limits(self):
         with self.assertRaises(ValueError) as cm:
             format(c, ".%sf" % (INT_MAX + 1))
 
+    @support.cpython_only
+    def test_precision_near_int_max(self):
+        # gh-158446: Precisions just below INT_MAX are rejected before any
+        # output buffer size is computed from them.
+        from _testcapi import INT_MAX
+
+        f = 1e300
+        c = complex(f)
+        for prec in (INT_MAX, INT_MAX - 1023):
+            for code in "feg":
+                spec = ".%d%s" % (prec, code)
+                with self.subTest(spec=spec):
+                    with self.assertRaises(ValueError):
+                        format(f, spec)
+                    with self.assertRaises(ValueError):
+                        format(c, spec)
+                    with self.assertRaises(ValueError):
+                        ("%" + spec) % f
+                    with self.assertRaises(ValueError):
+                        ("%" + spec).encode() % f
+
     def test_g_format_has_no_trailing_zeros(self):
         # regression test for bugs.python.org/issue40780
         self.assertEqual("%.3g" % 1505.0, "1.5e+03")

Lib/test/test_format.py

@@ -2103,6 +2103,86 @@ def test_unwrap(self):
         c_in.write(s_out.read())
         client.unwrap()
 
+    def test_sni_callback_context_released_and_callback_raises(self):
+        # Variant of the test below without a HelloRetryRequest: the callback
+        # switches the connection to another context, drops the last
+        # references to the context that carries it, and raises.  The C
+        # callback must not touch that context after the Python callback
+        # returned.
+        client_ctx, server_ctx, hostname = testing_context()
+        leaf_ctx = server_ctx
+
+        def sni_cb(sslobj, server_name, ctx):
+            sslobj.context = leaf_ctx
+            del ctx
+            raise LookupError("no certificate for " + repr(server_name))
+
+        def make_server():
+            dispatch_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+            dispatch_ctx.load_cert_chain(SIGNED_CERTFILE)
+            dispatch_ctx.sni_callback = sni_cb
+            s_in, s_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+            server = dispatch_ctx.wrap_bio(s_in, s_out, server_side=True)
+            return server, s_in, s_out
+
+        server, s_in, s_out = make_server()
+        c_in, c_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+        client = client_ctx.wrap_bio(c_in, c_out, server_hostname=hostname)
+        with self.assertRaises(ssl.SSLWantReadError):
+            client.do_handshake()
+        s_in.write(c_out.read())
+        with support.catch_unraisable_exception() as cm:
+            with self.assertRaises(ssl.SSLError):
+                server.do_handshake()
+            self.assertIsInstance(cm.unraisable.exc_value, LookupError)
+        self.assertIs(server.context, leaf_ctx)
+
+    def test_sni_callback_context_released_before_second_client_hello(self):
+        # The SSLContext carrying sni_callback may be released by the
+        # application once the callback has switched the connection over to
+        # another context.  If the server then sends a HelloRetryRequest, the
+        # second ClientHello makes OpenSSL consult the original SSL_CTX's
+        # servername callback again; that must not use the deallocated
+        # SSLContext object.
+        client_ctx, leaf_ctx, hostname = testing_context()
+        calls = []
+
+        def make_server():
+            dispatch_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+            dispatch_ctx.load_cert_chain(SIGNED_CERTFILE)
+            # Force a HelloRetryRequest: the client offers an X25519 key
+            # share first, the server only accepts P-384.
+            dispatch_ctx.set_ecdh_curve("secp384r1")
+            def sni_cb(sslobj, server_name, ctx):
+                calls.append(server_name)
+                sslobj.context = leaf_ctx
+            dispatch_ctx.sni_callback = sni_cb
+            s_in, s_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+            server = dispatch_ctx.wrap_bio(s_in, s_out, server_side=True)
+            return server, s_in, s_out, weakref.ref(dispatch_ctx)
+
+        # After this only the C-level SSL object references dispatch_ctx.
+        server, s_in, s_out, dispatch_ref = make_server()
+        c_in, c_out = ssl.MemoryBIO(), ssl.MemoryBIO()
+        client = client_ctx.wrap_bio(c_in, c_out, server_hostname=hostname)
+        for _ in range(10):
+            for obj, out, peer_in in ((client, c_out, s_in),
+                                      (server, s_out, c_in)):
+                try:
+                    obj.do_handshake()
+                except ssl.SSLWantReadError:
+                    pass
+                if out.pending:
+                    peer_in.write(out.read())
+        client.do_handshake()
+        server.do_handshake()
+        support.gc_collect()
+        self.assertIsNone(dispatch_ref())
+        self.assertGreaterEqual(len(calls), 1)
+        self.assertEqual(calls[0], hostname)
+        self.assertIs(server.context, leaf_ctx)
+        self.assertIsNotNone(client.cipher())
+
 class SimpleBackgroundTests(unittest.TestCase):
     """Tests that connect to a simple server running in the background"""
 
@@ -5082,15 +5162,27 @@ def non_linux_skip_if_other_okay_error(self, err):
             return  # Expect the full test setup to always work on Linux.
         if (isinstance(err, ConnectionResetError) or
             (isinstance(err, OSError) and err.errno == errno.EINVAL) or
-            re.search('wrong.version.number', getattr(err, "reason", ""), re.I)):
+            re.search(
+                # Matches the following error messages:
+                # '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)'
+                # '[SSL: RECORD_LAYER_FAILURE] record layer failure (_ssl.c:1109)'
+                # '[SSL: HTTP_REQUEST] http request (_ssl.c:1143)'
+                r'wrong.version.number|record.layer.failure|http.request',
+                str(getattr(err, "reason", "")),
+                re.IGNORECASE,
+            )
+        ):
             # On Windows the TCP RST leads to a ConnectionResetError
             # (ECONNRESET) which Linux doesn't appear to surface to userspace.
             # If wrap_socket() winds up on the "if connected:" path and doing
-            # the actual wrapping... we get an SSLError from OpenSSL. Typically
-            # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario
-            # we're specifically trying to test. The way this test is written
-            # is known to work on Linux. We'll skip it anywhere else that it
-            # does not present as doing so.
+            # the actual wrapping... we get an SSLError from OpenSSL. This is
+            # typically WRONG_VERSION_NUMBER. The same happens on iOS, but
+            # RECORD_LAYER_FAILURE or HTTP_REQUEST is the error.
+            #
+            # While appropriate, these scenarios aren't what we're specifically
+            # trying to test. The way this test is written is known to work on
+            # Linux. We'll skip it anywhere else that it does not present as
+            # doing so.
             try:
                 self.skipTest(f"Could not recreate conditions on {sys.platform}:"
                               f" {err=}")

Lib/test/test_ssl.py

@@ -3977,9 +3977,15 @@ def test_sneaky_hardlink_fallback(self):
         for filter in 'tar', 'fully_trusted':
             with self.subTest(filter), self.check_context(arc.open(), filter):
                 if not os_helper.can_symlink():
-                    self.expect_file("a/t/dummy")
-                    self.expect_file("b/")
-                    self.expect_file("c/")
+                    if filter == 'fully_trusted' or sys.platform == "win32":
+                        self.expect_file("a/t/dummy")
+                        self.expect_file("b/")
+                        self.expect_file("c/")
+                    else:
+                        self.expect_exception(
+                            tarfile.LinkFallbackError,
+                            "link 'boom' would be extracted as a copy of "
+                            + "'c/escape', which was rejected")
                 else:
                     self.expect_file("a/t/dummy")
                     self.expect_file("b/")
@@ -4170,6 +4176,24 @@ def testing_filter(member, path):
             if sys.platform != 'win32':
                 self.assertFalse(path.stat().st_mode & stat.S_IWUSR)
 
+    def test_extract_filters_target_none(self):
+        # Test that when extract() falls back to extracting (rather than
+        # linking) a hardlink target, the member is skipped if the filter
+        # returns None.
+        with ArchiveMaker() as arc:
+            arc.add('a/b/s', symlink_to='../escape')
+            arc.add('q', hardlink_to='a/b/s')
+        def filter_unsafe_members(member, path):
+            try:
+                return tarfile.data_filter(member, path)
+            except tarfile.FilterError as error:
+                return None
+        with self.check_context(arc.open(), filter_unsafe_members):
+            if os_helper.can_symlink():
+                self.expect_file('a/b/s', symlink_to='../escape')
+            else:
+                self.expect_file('a/b/')  # symlink is not extracted
+
     def test_link_fallback_normalizes(self):
         # Make sure hardlink fallbacks work for non-normalized paths for all
         # filters