python 3.15.0rc2: split libpython shared output

anaconda.yaml

@@ -7,6 +7,7 @@ upstream_sources:
       from_pattern: ^v(\d+\.\d+\.\d+)$
       to_pattern: \1
     packages:
+      - libpython
       - libpython-static
       - python
       - python-freethreading
@@ -16,6 +17,7 @@ upstream_sources:
       url: https://www.python.org/ftp/python/
       regex: href="(3\.14\.\d+)/"
     packages:
+      - libpython
       - libpython-static
       - python
       - python-freethreading
@@ -27,6 +29,7 @@ upstream_sources:
       url: https://www.python.org/ftp/python/
       regex: href="(3\.13\.\d+)/"
     packages:
+      - libpython
       - libpython-static
       - python
       - python-freethreading
@@ -38,6 +41,7 @@ upstream_sources:
       url: https://www.python.org/ftp/python/
       regex: href="(3\.12\.\d+)/"
     packages:
+      - libpython
       - libpython-static
       - python
     feedstock_branch: 'main-3.12'
@@ -47,6 +51,7 @@ upstream_sources:
       url: https://www.python.org/ftp/python/
       regex: href="(3\.11\.\d+)/"
     packages:
+      - libpython
       - libpython-static
       - python
     feedstock_branch: 'main-3.11'

recipe/build_base.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -ex
+set -exo pipefail
 
 cd ${SRC_DIR}
 
@@ -336,28 +336,31 @@ pushd ${_buildd_static}
                        ${_DISABLE_SHARED} "${_PROFILE_TASK[@]}"
 popd
 
-if [[ "${CI}" == "travis" ]]; then
-  # Travis has issues with long logs
-  make -j${CPU_COUNT} -C ${_buildd_static} \
-       EXTRA_CFLAGS="${EXTRA_CFLAGS}" \
-       ${_MAKE_TARGET} "${_PROFILE_TASK[@]}" 2>&1 >make-static.log
-else
-  make -j${CPU_COUNT} -C ${_buildd_static} \
-       EXTRA_CFLAGS="${EXTRA_CFLAGS}" \
-       ${_MAKE_TARGET} "${_PROFILE_TASK[@]}" 2>&1 | tee make-static.log
-fi
+make -j${CPU_COUNT} -C ${_buildd_static} \
+      EXTRA_CFLAGS="${EXTRA_CFLAGS}" \
+      ${_MAKE_TARGET} "${_PROFILE_TASK[@]}" 2>&1 | tee make-static.log
 if rg "Failed to build these modules" make-static.log; then
   echo "(static) :: Failed to build some modules, check the log"
   exit 1
 fi
 
-if [[ "${CI}" == "travis" ]]; then
-  # Travis has issues with long logs
-  make -j${CPU_COUNT} -C ${_buildd_shared} \
-          EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 >make-shared.log
-else
-  make -j${CPU_COUNT} -C ${_buildd_shared} \
-          EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 | tee make-shared.log
+# The macOS dylib's install name points at PREFIX/lib, but it is built in
+# build-shared first. Make it available at that path while PGO runs Python.
+if [[ ${target_platform} == osx-* ]]; then
+  shared_lib_link=${PREFIX}/lib/libpython${VERABI}.dylib
+  if [[ -e ${shared_lib_link} || -L ${shared_lib_link} ]]; then
+    echo "Refusing to replace existing ${shared_lib_link}"
+    exit 1
+  fi
+  ln -s "${SRC_DIR}/${_buildd_shared}/libpython${VERABI}.dylib" "${shared_lib_link}"
+  trap 'rm -f "${shared_lib_link}"' EXIT
+fi
+
+make -j${CPU_COUNT} -C ${_buildd_shared} \
+        EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 | tee make-shared.log
+if [[ ${target_platform} == osx-* ]]; then
+  rm -f "${shared_lib_link}"
+  trap - EXIT
 fi
 if rg "Failed to build these modules" make-shared.log; then
   echo "(shared) :: Failed to build some modules, check the log"
@@ -386,42 +389,9 @@ if [[ ${_OPTIMIZED} == yes ]]; then
     _FLAGS_REPLACE+=("")
   done
 fi
-# Install the shared library (for people who embed Python only, e.g. GDB).
-# Linking module extensions to this on Linux is redundant (but harmless).
-# Linking module extensions to this on Darwin is harmful (multiply defined symbols).
-shopt -s extglob
-cp -pf ${_buildd_shared}/libpython*${SHLIB_EXT}!(.lto) ${PREFIX}/lib/
-shopt -u extglob
-if [[ ${target_platform} =~ .*linux.* ]]; then
-  ln -sf ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}.1.0 ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}
-fi
-
-# create libpython3.dylib; linux gets libpython3.so from upstream's Makefile,
-# macOS has no upstream rule — build the stable-ABI re-export dylib ourselves
-# (same as CF install_shared.sh). Release only, matching linux.
-if [[ "$target_platform" == osx-* && ${PY_INTERP_DEBUG} == no ]]; then
-  # need to filter out windows-specific symbols & PyOS_CheckStack from
-  # https://github.com/python/cpython/blob/main/Doc/data/stable_abi.dat
-  awk -F',' '
-    ($1 == "func" || $1 == "data") &&
-    $4 != "on Windows" &&
-    $2 != "PyOS_CheckStack" {
-      print "_" $2
-    }
-  ' ${SRC_DIR}/Doc/data/stable_abi.dat > ${_buildd_shared}/stable_abi_exports.txt
-
-  $CC -dynamiclib \
-   -install_name @rpath/libpython3.dylib \
-   -compatibility_version 3.0 -current_version ${VER}.0 \
-   -Wl,-reexport_library,${PREFIX}/lib/libpython${VERABI}.dylib \
-   -Wl,-exported_symbols_list,${_buildd_shared}/stable_abi_exports.txt \
-   -o ${PREFIX}/lib/libpython3.dylib
-fi
-
-# AR: keep sysconfig from the *static* build (same as CF install_base.sh).
-# A shared-build sysconfig made python3-config --embed emit -lpython3.15, so
-# libpython-static tests linked the dylib and dyld aborted on osx-arm64.
-SYSCONFIG=$(find ${_buildd_static}/$(cat ${_buildd_static}/pybuilddir.txt) -name "_sysconfigdata*.py" -print0)
+# Use sysconfigdata from the shared build, as we want packages to prefer
+# linking against the shared library. Issue #565.
+SYSCONFIG=$(find ${_buildd_shared}/$(cat ${_buildd_shared}/pybuilddir.txt) -name "_sysconfigdata*.py" -print0)
 cat ${SYSCONFIG} | ${SYS_PYTHON} "${RECIPE_DIR}"/replace-word-pairs.py \
   "${_FLAGS_REPLACE[@]}"  \
     > ${PREFIX}/lib/python${VERABI}/$(basename ${SYSCONFIG})

recipe/install_shared.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -ex
+
+_buildd_shared=build-shared
+if [[ ${PY_INTERP_DEBUG} == yes ]]; then
+  DBG=d
+else
+  DBG=
+fi
+if [[ ${PY_FREETHREADING} == yes ]]; then
+  # This Python will not be usable with non-free threading Python modules.
+  THREAD=t
+else
+  THREAD=
+fi
+
+VER=${PKG_VERSION%.*}
+ABIFLAGS=${DBG}${THREAD}
+VERABI=${VER}${THREAD}${DBG}
+VERABI_NO_DBG=${VER}${THREAD}
+
+# Install the shared library (for people who embed Python only, e.g. GDB).
+# Linking module extensions to this on Linux is redundant (but harmless).
+# Linking module extensions to this on Darwin is harmful (multiply defined symbols).
+shopt -s extglob
+cp -pf ${_buildd_shared}/libpython*${SHLIB_EXT}!(.lto) ${PREFIX}/lib/
+shopt -u extglob
+if [[ ${target_platform} =~ .*linux.* ]]; then
+  ln -sf ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}.1.0 ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}
+fi
+if [[ ${PY_INTERP_DEBUG} == yes ]]; then
+  ln -s ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} ${PREFIX}/lib/libpython${VERABI_NO_DBG}${SHLIB_EXT}
+fi
+
+# create libpython3.dylib; linux builds only add this in release mode, do the same on osx
+if [[ "$target_platform" == osx-* && ${PY_INTERP_DEBUG} == no ]]; then
+  # need to filter out windows-specific symbols & PyOS_CheckStack from
+  # https://github.com/python/cpython/blob/main/Doc/data/stable_abi.dat
+  awk -F',' '
+    ($1 == "func" || $1 == "data") &&
+    $4 != "on Windows" &&
+    $2 != "PyOS_CheckStack" {
+      print "_" $2
+    }
+  ' Doc/data/stable_abi.dat > stable_abi_exports.txt
+
+  $CC -dynamiclib \
+   -install_name @rpath/libpython3.dylib \
+   -compatibility_version 3.0 -current_version ${PY_VER}.0 \
+   -Wl,-reexport_library,${PREFIX}/lib/libpython${VERABI}.dylib \
+   -Wl,-exported_symbols_list,stable_abi_exports.txt \
+   -o ${PREFIX}/lib/libpython3.dylib
+fi

recipe/build_static.sh → recipe/install_static.sh

@@ -1,22 +1,24 @@
 #!/bin/bash
 set -ex
 
-cd ${SRC_DIR}
-
 _buildd_static=build-static
 _buildd_shared=build-shared
-if [[ ${DEBUG_PY} == yes ]]; then
+if [[ ${PY_INTERP_DEBUG} == yes ]]; then
   DBG=d
 else
   DBG=
 fi
-if [[ ${PY_GIL_DISABLED} == yes ]]; then
+if [[ ${PY_FREETHREADING} == yes ]]; then
+  # This Python will not be usable with non-free threading Python modules.
   THREAD=t
 else
   THREAD=
 fi
+
 VER=${PKG_VERSION%.*}
-VERABI=${VER}${DBG}${THREAD}
+ABIFLAGS=${DBG}${THREAD}
+VERABI=${VER}${THREAD}${DBG}
+VERABI_NO_DBG=${VER}${THREAD}
 
 case "$target_platform" in
   linux-64)

recipe/meta.yaml

@@ -4,7 +4,7 @@
 {% set ver2 = '.'.join(version.split('.')[0:2]) %}
 {% set ver2nd = ''.join(version.split('.')[0:2]) %}
 {% set ver3nd = ''.join(version.split('.')[0:3]) %}
-{% set build_number = 0 %}
+{% set build_number = 1 %}
 {% set channel_targets = ('abc', 'def')  %}
 # this is just for the initial build, to break dependencies with python -> pip -> libpython-static
 {% set bootstrap = "false" %}
@@ -255,7 +255,6 @@ outputs:
       files:
         - tests/cmake/*
         - tests/cython/*
-        - tests/prefix-replacement/*
         - run_test.py
       commands:
         - echo on  # [win]
@@ -281,12 +280,11 @@ outputs:
         - _CONDA_PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_x86_64_conda_linux_gnu python -c "import sysconfig; print(sysconfig.get_config_var('CC'))"   # [linux64]
         # check for unreplaced @ symbols in sysconfig files, excluding e.g. '"$@".lto'
         - for f in ${CONDA_PREFIX}/lib/python*/_sysconfig*.py; do echo "Checking $f:"; if [[ `rg @[^@]*@ $f` ]]; then echo "FAILED ON $f"; cat $f; exit 1; fi; done  # [linux64 or osx]
-        # AR: no separate libpython output (CF has one). Shared libpython*.so/.dylib
-        # ships in this package, so do not assert their absence. Static libs must not.
+        - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.so           # [unix]
         - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.a            # [unix]
+        - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.so           # [linux]
+        - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.dylib        # [osx]
         - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.nolto.a      # [unix]
-        # stable-ABI re-export dylib (upstream has no macOS rule; CF install_shared.sh parity)
-        - test -f ${PREFIX}/lib/libpython3.dylib                          # [osx]
         # https://github.com/conda-forge/python-feedstock/issues/384
         - if exist %PREFIX%\\Scripts\\pydoc exit 1  # [win]
         - if exist %PREFIX%\\Scripts\\idle exit 1  # [win]
@@ -300,9 +298,6 @@ outputs:
         # libs\ kept for link-time compat (CFEP-27 section 6, conda-forge PR 918)
         - if not exist %PREFIX%\\libs\\python{{ ver2nd }}{{ abi_suffix }}.lib exit 1  # [win]
         - pushd tests
-        - pushd prefix-replacement    # [unix]
-        - bash build-and-test.sh      # [unix]
-        - popd    # [unix]
         - pushd cmake
         - cmake -GNinja -DPY_VER={{ version }} -DEXTRA_COMPONENTS="Development.Module" --debug-find --trace --debug-output --debug-trycompile .
         - popd
@@ -314,20 +309,76 @@ outputs:
         # Test for wide character supported via ncursesw
         - TERM=xterm >/dev/null python -c "import curses; scr = curses.initscr(); curses.unget_wch('x'); assert 'x' ==  scr.get_wch()"  # [unix]
 
+  # this output is empty on windows
+  - name: libpython
+    script: install_shared.sh   # [unix]
+    build:
+      number: {{ build_number }}
+      activate_in_script: true
+      ignore_run_exports:
+        - python_abi
+      ignore_run_exports_from:   # [unix]
+        - {{ compiler('cxx') }}  # [unix]
+      script_env:
+        - PY_INTERP_DEBUG={{ py_interp_debug }}
+        - PY_FREETHREADING={{ py_gil_disabled }}
+      string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}{{ linkage_nature }}{{ debug }}_{{ abi_tag }}
+      run_exports:                                                           # [unix]
+        weak:                                                                # [unix]
+          - {{ pin_subpackage("libpython", min_pin="x.x", max_pin="x") }}    # [unix]
+    requirements:
+      build:
+        - {{ compiler('c') }}
+        - {{ stdlib('c') }}
+        - {{ compiler('cxx') }}
+      host:
+        - {{ pin_subpackage('python', exact=True) }}
+      run:
+        - {{ pin_subpackage('python', exact=True) }}
+    test:
+      files:
+        - tests/prefix-replacement/*
+      requires:
+        - {{ stdlib('c') }}
+        - {{ compiler('c') }}
+        - {{ compiler('cxx') }}
+        - ripgrep
+      commands:
+        - VER=${PKG_VERSION%.*}    # [unix]
+        - VERABI=${VER}            # [unix]
+{% if py_gil_disabled == "yes" %}
+        - VERABI=${VERABI}t        # [unix]
+{% endif %}
+{% if py_interp_debug == "yes" %}
+        - VERABI=${VERABI}d        # [unix]
+{% endif %}
+        - test -f ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}       # [unix]
+{% if py_interp_debug == "no" %}
+        - test -f ${PREFIX}/lib/libpython3${SHLIB_EXT}               # [unix]
+{% endif %}
+        - pushd tests                 # [unix]
+        - pushd prefix-replacement    # [unix]
+        - bash build-and-test.sh      # [unix]
+        - popd                        # [unix]
+        - popd                        # [unix]
+
 {% if bootstrap != "true" %}
   - name: libpython-static
-    script: build_static.sh  # [unix]
-    script: build_static.bat  # [win]
+    script: install_static.sh  # [unix]
     build:
       number: {{ build_number }}
       force_use_keys:
         - gil_type
       activate_in_script: true
       ignore_run_exports:
         - python_abi
+      ignore_run_exports_from:   # [unix]
+        - {{ compiler('cxx') }}  # [unix]
       string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}{{ linkage_nature }}{{ debug }}_{{ abi_tag }}
       script_env:
         - PY_GIL_DISABLED={{ py_gil_disabled }}
+        - PY_INTERP_DEBUG={{ py_interp_debug }}
+        - PY_FREETHREADING={{ py_gil_disabled }}
     requirements:
       build:
         - {{ stdlib('c') }}
@@ -368,8 +419,9 @@ outputs:
       skip: True
 {% endif %}
     requirements:
-      - python {{ version }}{{ dev }}.*
-      - python_abi * *_{{ abi_tag }}
+      run:
+        - python {{ version }}{{ dev }}.*
+        - python_abi * *_{{ abi_tag }}
 
   - name: python-gil
     build:
@@ -378,8 +430,9 @@ outputs:
       skip: True
 {% endif %}
     requirements:
-      - python {{ version }}{{ dev }}.*
-      - python_abi * *_{{ abi_tag }}
+      run:
+        - python {{ version }}{{ dev }}.*
+        - python_abi * *_{{ abi_tag }}
 
   - name: python-jit
     script: install_jit_activation.sh   # [not win]
@@ -391,9 +444,10 @@ outputs:
       skip: true  # [py<313]
       noarch: generic
     requirements:
-      - python {{ version }}{{ dev }}
-      - __unix   # [unix]
-      - __win    # [win]
+      run:
+        - python {{ version }}{{ dev }}
+        - __unix   # [unix]
+        - __win    # [win]
     test:
       files:
         - run_test_jit.py