Use standard extension configuration for CPython 3.12+

cpython-unix/Makefile

@@ -188,6 +188,9 @@ $(OUTDIR)/libedit-$(LIBEDIT_VERSION)-$(PACKAGE_SUFFIX).tar: $(LIBEDIT_DEPENDS)
 $(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-patchelf.sh
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) patchelf
 
+$(OUTDIR)/pkgconf-$(PKGCONF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-pkgconf.sh
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) pkgconf
+
 $(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-sqlite.sh
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) sqlite
 
@@ -267,6 +270,7 @@ PYTHON_DEPENDS_$(1) := \
     $$(if $$(NEED_OPENSSL_1_1),$$(OUTDIR)/openssl-1.1-$$(OPENSSL_1.1_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_OPENSSL_3_5),$$(OUTDIR)/openssl-3.5-$$(OPENSSL_3.5_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_PATCHELF),$$(OUTDIR)/patchelf-$$(PATCHELF_VERSION)-$$(PACKAGE_SUFFIX).tar) \
+    $$(if $$(NEED_PKGCONF),$$(OUTDIR)/pkgconf-$$(PKGCONF_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \

cpython-unix/build-cpython.sh

@@ -15,6 +15,21 @@ export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib
 # Ensure that `pkg-config` invocations include the static libraries
 export PKG_CONFIG="pkg-config --static"
 
+# Dependency pkg-config files use /tools/deps as their prefix. macOS builds
+# extract dependencies into a temporary directory, so have pkgconf relocate
+# their prefix based on the location of each .pc file.
+if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
+    export PKG_CONFIG="${PKG_CONFIG} --define-prefix"
+
+    if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
+        # On macOS, CPython prefers the system libffi and provides no way to
+        # select a non-system version. Disable its system-library probe so
+        # _ctypes uses the bundled libffi instead.
+        # https://github.com/python/cpython/issues/155813
+        export ac_cv_lib_ffi_ffi_call=no
+    fi
+fi
+
 # configure somehow has problems locating llvm-profdata even though it is in
 # PATH. The macro it is using allows us to specify its path via an
 # environment variable.
@@ -203,23 +218,12 @@ if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]]; then
     patch -p1 -i "${ROOT}/patch-posixmodule-remove-system.patch"
 fi
 
-# Python 3.11 has configure support for configuring extension modules. We really,
-# really, really want to use this feature because it looks promising. But at the
-# time we added this code the functionality didn't support all extension modules
-# nor did it easily support static linking, including static linking of extra
-# libraries (which appears to be a limitation of `makesetup`). So for now we
-# disable the functionality and require our auto-generated Setup.local to provide
-# everything.
-if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then
-    if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
-        # This sets MODULE_<NAME>_STATE=disabled in the Makefile for all extension
-        # modules that are not unavailable (n/a) based on the platform.
-        # Valid STATE variables are needed to create the _missing_stdlib_info.py
-        # file during the build in Python 3.15+
-        patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod-3.12.patch"
-    else
-        patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod.patch"
-    fi
+# CPython 3.11 does not configure every standard-library extension, so disable
+# its incomplete generated rules and supply all rules through Setup.local.
+# CPython 3.12+ uses Setup.stdlib with a small Setup.local for per-module
+# link-type overrides.
+if [ "${PYTHON_MAJMIN_VERSION}" = "3.11" ]; then
+    patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod.patch"
 
     # This hack also prevents the conditional definition of the pwd module in
     # Setup.bootstrap.in from working. So we remove that conditional.
@@ -247,17 +251,6 @@ else
     patch -p1 -i "${ROOT}/patch-pgo-file-pool-3.11.patch"
 fi
 
-# There's a post-build Python script that verifies modules were
-# built correctly. Ideally we'd invoke this. But our nerfing of
-# the configure-based module building and replacing it with our
-# own Setup-derived version completely breaks assumptions in this
-# script. So leave it off for now... at our own peril.
-if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then
-    patch -p1 -i "${ROOT}/patch-checksharedmods-disable-3.15.patch"
-elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
-    patch -p1 -i "${ROOT}/patch-checksharedmods-disable.patch"
-fi
-
 # CPython < 3.11 always linked against libcrypt. We backport part of
 # upstream commit be21706f3760bec8bd11f85ce02ed6792b07f51f to avoid this
 # behavior.
@@ -385,13 +378,6 @@ if [[ "${CC}" = "clang" && -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
     EXTRA_CONFIGURE_FLAGS="${EXTRA_CONFIGURE_FLAGS} --with-tail-call-interp"
 fi
 
-# On Python 3.12+ we need to link the special hacl library provided some SHA-256
-# implementations. Since we hack up the regular extension building mechanism, we
-# need to reinvent this wheel.
-if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
-    LDFLAGS="${LDFLAGS} -LModules/_hacl"
-fi
-
 # On PPC we need to prevent the glibc 2.22 __tls_get_addr_opt symbol
 # from being introduced to preserve runtime compatibility with older
 # glibc.
@@ -421,6 +407,23 @@ CONFIGURE_FLAGS="
     --without-ensurepip
     ${EXTRA_CONFIGURE_FLAGS}"
 
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
+    # Build standard-library extensions as built-ins by default. Setup.local
+    # overrides only the extensions that must remain shared or disabled.
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} MODULE_BUILDTYPE=static --enable-loadable-sqlite-extensions"
+fi
+
+if [[ "${PYTHON_MAJMIN_VERSION}" = "3.12" && "${PYBUILD_PLATFORM}" = macos* ]]; then
+    # CPython 3.12 always uses UNIVERSAL on macOS for libmpdec,
+    # while the bundled dependency requires CONFIG_64.
+    export LIBMPDEC_CFLAGS="-DCONFIG_64=1"
+fi
+
+if [[ "${PYTHON_MAJMIN_VERSION}" = "3.12" && "${TARGET_TRIPLE}" == *-linux-* ]]; then
+    # The panelw archive needs ncursesw during configure's link probe.
+    export PANEL_CFLAGS="-I${TOOLS_PATH}/deps/include/ncursesw"
+    export PANEL_LIBS="-lpanelw -lncursesw"
+fi
 
 # Build a libpython3.x.so, but statically link the interpreter against
 # libpython.
@@ -641,13 +644,6 @@ if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64*
     PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format"
 fi
 
-# ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL,
-# but on x86-64 that requires v2 and v3 respectively, and on arm64 the performance is bad as noted
-# in the comments, so just don't even try. (We should check if we can make this conditional)
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
-    patch -p1 -i "${ROOT}/patch-python-configure-hacl-no-simd.patch"
-fi
-
 # We use ndbm on macOS and BerkeleyDB elsewhere.
 if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
     CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --with-dbmliborder=ndbm"
@@ -777,6 +773,12 @@ BOLT_COMMON_FLAGS="${BOLT_COMMON_FLAGS:-}" BOLT_APPLY_FLAGS="${BOLT_APPLY_FLAGS:
 # Supplement produced Makefile with our modifications.
 cat ../Makefile.extra >> Makefile
 
+if [[ "${PYBUILD_PLATFORM}" = macos* && -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]]; then
+    # configure-derived MODLIBS uses plain -l flags. Hide symbols from bundled
+    # static dependency libraries in both libpython and the interpreter.
+    printf '\nMODLIBS := $(subst -l,-Xlinker -hidden-l,$(MODLIBS))\n' >> Makefile
+fi
+
 # Debian's PPC64LE GCC 6 defaults to PIE but selects the non-PIE startup object unless -pie is
 # passed explicitly. Add it only to LINKFORSHARED, which CPython uses when linking executables.
 # See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81170.
@@ -1112,9 +1114,6 @@ if linux_uapi_include_arch:
         "",
     )
 replace_in_all("-isystem %s/deps/linux-uapi/usr/include" % tools_path, "")
-# See https://github.com/python/cpython/issues/145810#issuecomment-4068139183
-replace_in_all("-LModules/_hacl", "")
-
 EOF
 
 ${BUILD_PYTHON} "${ROOT}/hack_sysconfig.py" "${ROOT}/out/python"

cpython-unix/build-pkgconf.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=$(pwd)
+
+export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH
+
+tar -xf "pkgconf-${PKGCONF_VERSION}.tar.xz"
+
+pushd "pkgconf-${PKGCONF_VERSION}"
+
+CC="${HOST_CC}" CXX="${HOST_CXX}" CFLAGS="${EXTRA_HOST_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_HOST_CFLAGS} -fPIC" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \
+    --build="${BUILD_TRIPLE}" \
+    --prefix=/tools/host \
+    --disable-shared \
+    --disable-dependency-tracking \
+    --with-system-libdir=/usr/lib \
+    --with-system-includedir=/usr/include
+
+make -j "${NUM_CPUS}"
+make -j "${NUM_CPUS}" install DESTDIR="${ROOT}/out"
+
+ln -s pkgconf "${ROOT}/out/tools/host/bin/pkg-config"
+
+"${ROOT}/out/tools/host/bin/pkg-config" --version

cpython-unix/build-tcl.sh

@@ -83,6 +83,25 @@ export CPPFLAGS="${CFLAGS}"
 # Initialize it so that package configure does not require an installed Tcl 9.
 export ZIPFS_BUILD=0
 
+# macOS uses SDK zlib, which provides -lz but no zlib.pc. Remove Tcl's
+# pkg-config requirement while retaining its Libs.private: -lz linkage.
+if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
+    patch -p1 <<'EOF'
+diff --git a/tcl.pc.in b/tcl.pc.in
+--- a/tcl.pc.in
++++ b/tcl.pc.in
+@@ -9,7 +9,7 @@ Name: Tool Command Language
+ Description: Tcl is a powerful, easy-to-learn dynamic programming language, suitable for a wide range of uses.
+ URL: https://www.tcl-lang.org/
+ Version: @TCL_VERSION@@TCL_PATCH_LEVEL@
+-Requires.private: @TCL_PC_REQUIRES_PRIVATE@ zlib >= 1.2.3
++Requires.private: @TCL_PC_REQUIRES_PRIVATE@
+ Libs: -L${libdir} @TCL_LIB_FLAG@ @TCL_STUB_LIB_FLAG@
+ Libs.private: @TCL_LIBS@
+ Cflags: -I${includedir} @TCL_PC_CFLAGS@
+EOF
+fi
+
 ./configure \
     --build="${BUILD_TRIPLE}" \
     --host="${TARGET_TRIPLE}" \

cpython-unix/build-tk.sh

@@ -31,6 +31,22 @@ if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
 else
     LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL"
     EXTRA_CONFIGURE_FLAGS="--x-includes=${TOOLS_PATH}/deps/include --x-libraries=${TOOLS_PATH}/deps/lib"
+
+    # Tk's pkg-config metadata omits its private X11 dependency. Add it so
+    # pkg-config --static also resolves X11's xcb and Xau dependencies.
+    patch -p1 <<'EOF'
+diff --git a/tk.pc.in b/tk.pc.in
+--- a/tk.pc.in
++++ b/tk.pc.in
+@@ -10,6 +10,7 @@ Description: Tk is a cross-platform graphical user interface toolkit, the stand
+ URL: https://www.tcl-lang.org/
+ Version: @TK_VERSION@@TK_PATCH_LEVEL@
+ Requires: tcl >= 9.0
++Requires.private: x11
+ Libs: -L${libdir} @TK_LIB_FLAG@ @TK_STUB_LIB_FLAG@
+ Libs.private: @XFT_LIBS@ @XLIBSW@
+ Cflags: -I${includedir}
+EOF
 fi
 
 CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \

cpython-unix/build.py

@@ -1175,6 +1175,7 @@ def main():
             "ncurses",
             "openssl-3.5",
             "patchelf",
+            "pkgconf",
             "sqlite",
             "tcl",
             "uuid",
@@ -1185,7 +1186,7 @@ def main():
             "zlib",
             "zstd",
         ):
-            tools_path = "host" if action in ("m4", "patchelf") else "deps"
+            tools_path = "host" if action in ("m4", "patchelf", "pkgconf") else "deps"
             extra_archives = {
                 "tcl": {"zlib"},
             }.get(action)

cpython-unix/extension-modules.yml

@@ -725,6 +725,20 @@ _testcapi:
     - source: _testcapi/weakref.c
       minimum-python-version: "3.13"
 
+_testclinic:
+  minimum-python-version: '3.12'
+  disabled-targets:
+    - .*
+  sources:
+    - _testclinic.c
+
+_testclinic_limited:
+  minimum-python-version: '3.13'
+  disabled-targets:
+    - .*
+  sources:
+    - _testclinic_limited.c
+
 _testexternalinspection:
   minimum-python-version: '3.13'
   maximum-python-version: '3.13'
@@ -1104,6 +1118,13 @@ xxlimited_35:
   disabled-targets:
     - .*
 
+xxlimited_3_13:
+  minimum-python-version: '3.15'
+  disabled-targets:
+    - .*
+  sources:
+    - xxlimited_3_13.c
+
 xxsubtype:
   setup-enabled-conditional:
     - enabled: true

cpython-unix/targets.yml

@@ -108,6 +108,7 @@ aarch64-apple-darwin:
     - m4
     - mpdecimal
     - openssl-3.5
+    - pkgconf
     - sqlite
     - tcl
     - tk
@@ -659,6 +660,7 @@ x86_64-apple-darwin:
     - m4
     - mpdecimal
     - openssl-3.5
+    - pkgconf
     - sqlite
     - tcl
     - tk