diff --git a/anaconda.yaml b/anaconda.yaml index c98639f2f..c53fbf835 100644 --- a/anaconda.yaml +++ b/anaconda.yaml @@ -7,6 +7,7 @@ upstream_sources: from_pattern: ^v(\d+\.\d+\.\d+)$ to_pattern: \1 packages: + - libpython - libpython-static - python - python-freethreading @@ -16,6 +17,7 @@ upstream_sources: url: https://www.python.org/ftp/python/ regex: href="(3\.14\.\d+)/" packages: + - libpython - libpython-static - python - python-freethreading @@ -27,6 +29,7 @@ upstream_sources: url: https://www.python.org/ftp/python/ regex: href="(3\.13\.\d+)/" packages: + - libpython - libpython-static - python - python-freethreading @@ -38,6 +41,7 @@ upstream_sources: url: https://www.python.org/ftp/python/ regex: href="(3\.12\.\d+)/" packages: + - libpython - libpython-static - python feedstock_branch: 'main-3.12' @@ -47,6 +51,7 @@ upstream_sources: url: https://www.python.org/ftp/python/ regex: href="(3\.11\.\d+)/" packages: + - libpython - libpython-static - python feedstock_branch: 'main-3.11' diff --git a/recipe/build_base.sh b/recipe/build_base.sh index b95b1a704..849e82122 100644 --- a/recipe/build_base.sh +++ b/recipe/build_base.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -ex +set -exo pipefail cd ${SRC_DIR} @@ -336,28 +336,31 @@ pushd ${_buildd_static} ${_DISABLE_SHARED} "${_PROFILE_TASK[@]}" popd -if [[ "${CI}" == "travis" ]]; then - # Travis has issues with long logs - make -j${CPU_COUNT} -C ${_buildd_static} \ - EXTRA_CFLAGS="${EXTRA_CFLAGS}" \ - ${_MAKE_TARGET} "${_PROFILE_TASK[@]}" 2>&1 >make-static.log -else - make -j${CPU_COUNT} -C ${_buildd_static} \ - EXTRA_CFLAGS="${EXTRA_CFLAGS}" \ - ${_MAKE_TARGET} "${_PROFILE_TASK[@]}" 2>&1 | tee make-static.log -fi +make -j${CPU_COUNT} -C ${_buildd_static} \ + EXTRA_CFLAGS="${EXTRA_CFLAGS}" \ + ${_MAKE_TARGET} "${_PROFILE_TASK[@]}" 2>&1 | tee make-static.log if rg "Failed to build these modules" make-static.log; then echo "(static) :: Failed to build some modules, check the log" exit 1 fi -if [[ "${CI}" == "travis" ]]; then - # Travis has issues with long logs - make -j${CPU_COUNT} -C ${_buildd_shared} \ - EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 >make-shared.log -else - make -j${CPU_COUNT} -C ${_buildd_shared} \ - EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 | tee make-shared.log +# The macOS dylib's install name points at PREFIX/lib, but it is built in +# build-shared first. Make it available at that path while PGO runs Python. +if [[ ${target_platform} == osx-* ]]; then + shared_lib_link=${PREFIX}/lib/libpython${VERABI}.dylib + if [[ -e ${shared_lib_link} || -L ${shared_lib_link} ]]; then + echo "Refusing to replace existing ${shared_lib_link}" + exit 1 + fi + ln -s "${SRC_DIR}/${_buildd_shared}/libpython${VERABI}.dylib" "${shared_lib_link}" + trap 'rm -f "${shared_lib_link}"' EXIT +fi + +make -j${CPU_COUNT} -C ${_buildd_shared} \ + EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 | tee make-shared.log +if [[ ${target_platform} == osx-* ]]; then + rm -f "${shared_lib_link}" + trap - EXIT fi if rg "Failed to build these modules" make-shared.log; then echo "(shared) :: Failed to build some modules, check the log" @@ -386,42 +389,9 @@ if [[ ${_OPTIMIZED} == yes ]]; then _FLAGS_REPLACE+=("") done fi -# Install the shared library (for people who embed Python only, e.g. GDB). -# Linking module extensions to this on Linux is redundant (but harmless). -# Linking module extensions to this on Darwin is harmful (multiply defined symbols). -shopt -s extglob -cp -pf ${_buildd_shared}/libpython*${SHLIB_EXT}!(.lto) ${PREFIX}/lib/ -shopt -u extglob -if [[ ${target_platform} =~ .*linux.* ]]; then - ln -sf ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}.1.0 ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} -fi - -# create libpython3.dylib; linux gets libpython3.so from upstream's Makefile, -# macOS has no upstream rule — build the stable-ABI re-export dylib ourselves -# (same as CF install_shared.sh). Release only, matching linux. -if [[ "$target_platform" == osx-* && ${PY_INTERP_DEBUG} == no ]]; then - # need to filter out windows-specific symbols & PyOS_CheckStack from - # https://github.com/python/cpython/blob/main/Doc/data/stable_abi.dat - awk -F',' ' - ($1 == "func" || $1 == "data") && - $4 != "on Windows" && - $2 != "PyOS_CheckStack" { - print "_" $2 - } - ' ${SRC_DIR}/Doc/data/stable_abi.dat > ${_buildd_shared}/stable_abi_exports.txt - - $CC -dynamiclib \ - -install_name @rpath/libpython3.dylib \ - -compatibility_version 3.0 -current_version ${VER}.0 \ - -Wl,-reexport_library,${PREFIX}/lib/libpython${VERABI}.dylib \ - -Wl,-exported_symbols_list,${_buildd_shared}/stable_abi_exports.txt \ - -o ${PREFIX}/lib/libpython3.dylib -fi - -# AR: keep sysconfig from the *static* build (same as CF install_base.sh). -# A shared-build sysconfig made python3-config --embed emit -lpython3.15, so -# libpython-static tests linked the dylib and dyld aborted on osx-arm64. -SYSCONFIG=$(find ${_buildd_static}/$(cat ${_buildd_static}/pybuilddir.txt) -name "_sysconfigdata*.py" -print0) +# Use sysconfigdata from the shared build, as we want packages to prefer +# linking against the shared library. Issue #565. +SYSCONFIG=$(find ${_buildd_shared}/$(cat ${_buildd_shared}/pybuilddir.txt) -name "_sysconfigdata*.py" -print0) cat ${SYSCONFIG} | ${SYS_PYTHON} "${RECIPE_DIR}"/replace-word-pairs.py \ "${_FLAGS_REPLACE[@]}" \ > ${PREFIX}/lib/python${VERABI}/$(basename ${SYSCONFIG}) diff --git a/recipe/build_static.bat b/recipe/build_static.bat deleted file mode 100644 index e69de29bb..000000000 diff --git a/recipe/install_shared.sh b/recipe/install_shared.sh new file mode 100755 index 000000000..2b87be49f --- /dev/null +++ b/recipe/install_shared.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -ex + +cd ${SRC_DIR} + +_buildd_shared=build-shared +if [[ ${PY_INTERP_DEBUG} == yes ]]; then + DBG=d +else + DBG= +fi +if [[ ${PY_FREETHREADING} == yes ]]; then + # This Python will not be usable with non-free threading Python modules. + THREAD=t +else + THREAD= +fi + +VER=${PKG_VERSION%.*} +ABIFLAGS=${DBG}${THREAD} +VERABI=${VER}${THREAD}${DBG} +VERABI_NO_DBG=${VER}${THREAD} + +# Install the shared library (for people who embed Python only, e.g. GDB). +# Linking module extensions to this on Linux is redundant (but harmless). +# Linking module extensions to this on Darwin is harmful (multiply defined symbols). +shopt -s extglob +mkdir -p ${PREFIX}/lib +cp -pf ${_buildd_shared}/libpython*${SHLIB_EXT}!(.lto) ${PREFIX}/lib/ +shopt -u extglob +if [[ ${target_platform} =~ .*linux.* ]]; then + ln -sf ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}.1.0 ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} +fi +if [[ ${PY_INTERP_DEBUG} == yes ]]; then + ln -s ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} ${PREFIX}/lib/libpython${VERABI_NO_DBG}${SHLIB_EXT} +fi + +# create libpython3.dylib; linux builds only add this in release mode, do the same on osx +if [[ "$target_platform" == osx-* && ${PY_INTERP_DEBUG} == no ]]; then + # need to filter out windows-specific symbols & PyOS_CheckStack from + # https://github.com/python/cpython/blob/main/Doc/data/stable_abi.dat + awk -F',' ' + ($1 == "func" || $1 == "data") && + $4 != "on Windows" && + $2 != "PyOS_CheckStack" { + print "_" $2 + } + ' Doc/data/stable_abi.dat > stable_abi_exports.txt + + $CC -dynamiclib \ + -install_name @rpath/libpython3.dylib \ + -compatibility_version 3.0 -current_version ${PY_VER}.0 \ + -Wl,-reexport_library,${PREFIX}/lib/libpython${VERABI}.dylib \ + -Wl,-exported_symbols_list,stable_abi_exports.txt \ + -o ${PREFIX}/lib/libpython3.dylib +fi diff --git a/recipe/build_static.sh b/recipe/install_static.sh old mode 100644 new mode 100755 similarity index 82% rename from recipe/build_static.sh rename to recipe/install_static.sh index 9b5f39083..3b2c31f53 --- a/recipe/build_static.sh +++ b/recipe/install_static.sh @@ -1,22 +1,24 @@ #!/bin/bash set -ex -cd ${SRC_DIR} - _buildd_static=build-static _buildd_shared=build-shared -if [[ ${DEBUG_PY} == yes ]]; then +if [[ ${PY_INTERP_DEBUG} == yes ]]; then DBG=d else DBG= fi -if [[ ${PY_GIL_DISABLED} == yes ]]; then +if [[ ${PY_FREETHREADING} == yes ]]; then + # This Python will not be usable with non-free threading Python modules. THREAD=t else THREAD= fi + VER=${PKG_VERSION%.*} -VERABI=${VER}${DBG}${THREAD} +ABIFLAGS=${DBG}${THREAD} +VERABI=${VER}${THREAD}${DBG} +VERABI_NO_DBG=${VER}${THREAD} case "$target_platform" in linux-64) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index cbb252aa1..50f97dee8 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -4,7 +4,7 @@ {% set ver2 = '.'.join(version.split('.')[0:2]) %} {% set ver2nd = ''.join(version.split('.')[0:2]) %} {% set ver3nd = ''.join(version.split('.')[0:3]) %} -{% set build_number = 0 %} +{% set build_number = 1 %} {% set channel_targets = ('abc', 'def') %} # this is just for the initial build, to break dependencies with python -> pip -> libpython-static {% set bootstrap = "false" %} @@ -255,7 +255,6 @@ outputs: files: - tests/cmake/* - tests/cython/* - - tests/prefix-replacement/* - run_test.py commands: - echo on # [win] @@ -281,12 +280,11 @@ outputs: - _CONDA_PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_x86_64_conda_linux_gnu python -c "import sysconfig; print(sysconfig.get_config_var('CC'))" # [linux64] # check for unreplaced @ symbols in sysconfig files, excluding e.g. '"$@".lto' - for f in ${CONDA_PREFIX}/lib/python*/_sysconfig*.py; do echo "Checking $f:"; if [[ `rg @[^@]*@ $f` ]]; then echo "FAILED ON $f"; cat $f; exit 1; fi; done # [linux64 or osx] - # AR: no separate libpython output (CF has one). Shared libpython*.so/.dylib - # ships in this package, so do not assert their absence. Static libs must not. + - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.so # [unix] - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.a # [unix] + - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.so # [linux] + - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.dylib # [osx] - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.nolto.a # [unix] - # stable-ABI re-export dylib (upstream has no macOS rule; CF install_shared.sh parity) - - test -f ${PREFIX}/lib/libpython3.dylib # [osx] # https://github.com/conda-forge/python-feedstock/issues/384 - if exist %PREFIX%\\Scripts\\pydoc exit 1 # [win] - if exist %PREFIX%\\Scripts\\idle exit 1 # [win] @@ -300,9 +298,6 @@ outputs: # libs\ kept for link-time compat (CFEP-27 section 6, conda-forge PR 918) - if not exist %PREFIX%\\libs\\python{{ ver2nd }}{{ abi_suffix }}.lib exit 1 # [win] - pushd tests - - pushd prefix-replacement # [unix] - - bash build-and-test.sh # [unix] - - popd # [unix] - pushd cmake - cmake -GNinja -DPY_VER={{ version }} -DEXTRA_COMPONENTS="Development.Module" --debug-find --trace --debug-output --debug-trycompile . - popd @@ -314,10 +309,63 @@ outputs: # Test for wide character supported via ncursesw - TERM=xterm >/dev/null python -c "import curses; scr = curses.initscr(); curses.unget_wch('x'); assert 'x' == scr.get_wch()" # [unix] + # this output is empty on windows + - name: libpython + script: install_shared.sh # [unix] + build: + skip: true # [win] + number: {{ build_number }} + activate_in_script: true + ignore_run_exports: + - python_abi + ignore_run_exports_from: # [unix] + - {{ compiler('cxx') }} # [unix] + script_env: + - PY_INTERP_DEBUG={{ py_interp_debug }} + - PY_FREETHREADING={{ py_gil_disabled }} + string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}{{ linkage_nature }}{{ debug }}_{{ abi_tag }} + run_exports: # [unix] + weak: # [unix] + - {{ pin_subpackage("libpython", min_pin="x.x", max_pin="x") }} # [unix] + requirements: + build: + - {{ compiler('c') }} + - {{ stdlib('c') }} + - {{ compiler('cxx') }} + - {{ pin_subpackage('python', exact=True) }} + host: [] + run: [] + test: + files: + - tests/prefix-replacement/* + requires: + - {{ pin_subpackage('python', exact=True) }} + - {{ stdlib('c') }} + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - ripgrep + commands: + - VER=${PKG_VERSION%.*} # [unix] + - VERABI=${VER} # [unix] +{% if py_gil_disabled == "yes" %} + - VERABI=${VERABI}t # [unix] +{% endif %} +{% if py_interp_debug == "yes" %} + - VERABI=${VERABI}d # [unix] +{% endif %} + - test -f ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} # [unix] +{% if py_interp_debug == "no" %} + - test -f ${PREFIX}/lib/libpython3${SHLIB_EXT} # [unix] +{% endif %} + - pushd tests # [unix] + - pushd prefix-replacement # [unix] + - bash build-and-test.sh # [unix] + - popd # [unix] + - popd # [unix] + {% if bootstrap != "true" %} - name: libpython-static - script: build_static.sh # [unix] - script: build_static.bat # [win] + script: install_static.sh # [unix] build: number: {{ build_number }} force_use_keys: @@ -325,9 +373,13 @@ outputs: activate_in_script: true ignore_run_exports: - python_abi + ignore_run_exports_from: # [unix] + - {{ compiler('cxx') }} # [unix] string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}{{ linkage_nature }}{{ debug }}_{{ abi_tag }} script_env: - PY_GIL_DISABLED={{ py_gil_disabled }} + - PY_INTERP_DEBUG={{ py_interp_debug }} + - PY_FREETHREADING={{ py_gil_disabled }} requirements: build: - {{ stdlib('c') }} @@ -368,8 +420,9 @@ outputs: skip: True {% endif %} requirements: - - python {{ version }}{{ dev }}.* - - python_abi * *_{{ abi_tag }} + run: + - python {{ version }}{{ dev }}.* + - python_abi * *_{{ abi_tag }} - name: python-gil build: @@ -378,8 +431,9 @@ outputs: skip: True {% endif %} requirements: - - python {{ version }}{{ dev }}.* - - python_abi * *_{{ abi_tag }} + run: + - python {{ version }}{{ dev }}.* + - python_abi * *_{{ abi_tag }} - name: python-jit script: install_jit_activation.sh # [not win] @@ -391,9 +445,10 @@ outputs: skip: true # [py<313] noarch: generic requirements: - - python {{ version }}{{ dev }} - - __unix # [unix] - - __win # [win] + run: + - python {{ version }}{{ dev }} + - __unix # [unix] + - __win # [win] test: files: - run_test_jit.py diff --git a/recipe/tests/prefix-replacement/build-and-test.sh b/recipe/tests/prefix-replacement/build-and-test.sh old mode 100644 new mode 100755 index 633b741a3..e6b8e889c --- a/recipe/tests/prefix-replacement/build-and-test.sh +++ b/recipe/tests/prefix-replacement/build-and-test.sh @@ -2,46 +2,51 @@ set -ex -if [[ "$PKG_NAME" == "libpython-static" ]]; then - # see bpo44182 for why -L${CONDA_PREFIX}/lib is added - ${CC} a.c $(python3-config --cflags) $(python3-config --embed --ldflags) -L${CONDA_PREFIX}/lib -o ${CONDA_PREFIX}/bin/embedded-python-static - if [[ "$target_platform" == linux-* ]]; then - if ${READELF} -d ${CONDA_PREFIX}/bin/embedded-python-static | rg libpython; then - echo "ERROR :: Embedded python linked to shared python library. It is expected to link to the static library." - fi - elif [[ "$target_platform" == osx-* ]]; then - if ${OTOOL} -l ${CONDA_PREFIX}/bin/embedded-python-static | rg libpython; then - echo "ERROR :: Embedded python linked to shared python library. It is expected to link to the static library." - fi - fi - ${CONDA_PREFIX}/bin/embedded-python-static - - # I thought this would prefer the shared library for Python. I was wrong: - # EMBED_LDFLAGS=$(python3-config --ldflags) - # re='^(.*)(-lpython[^ ]*)(.*)$' - # if [[ ${EMBED_LDFLAGS} =~ $re ]]; then - # EMBED_LDFLAGS="${BASH_REMATCH[1]} ${BASH_REMATCH[3]} -Wl,-Bdynamic ${BASH_REMATCH[2]}" - # fi - # ${CC} a.c $(python3-config --cflags) ${EMBED_LDFLAGS} -o ${CONDA_PREFIX}/bin/embedded-python-shared +case ${PKG_NAME} in + libpython) + # see bpo44182 for why -L${CONDA_PREFIX}/lib is added + ${CC} a.c $(python3-config --cflags) \ + $(python3-config --embed --ldflags) \ + -L${CONDA_PREFIX}/lib -Wl,-rpath,${CONDA_PREFIX}/lib \ + -o ${CONDA_PREFIX}/bin/embedded-python-shared - # Brute-force way of linking to the shared library, sorry! - rm -rf ${CONDA_PREFIX}/lib/libpython*.a -fi + if [[ "$target_platform" == linux-* ]]; then + if ! ${READELF} -d ${CONDA_PREFIX}/bin/embedded-python-shared | rg '\(NEEDED\).*\[libpython[^]]*\]'; then + echo "ERROR :: Embedded python linked to static python library. We tried to force it to use the shared library." + exit 1 + fi + elif [[ "$target_platform" == osx-* ]]; then + if ! ${OTOOL} -L ${CONDA_PREFIX}/bin/embedded-python-shared | rg '^[[:space:]]+[^[:space:]]*/libpython[^[:space:]]*\.dylib[[:space:]]'; then + echo "ERROR :: Embedded python linked to static python library. We tried to force it to use the shared library." + exit 1 + fi + fi + ${CONDA_PREFIX}/bin/embedded-python-shared + ;; -${CC} a.c $(python3-config --cflags) \ - $(python3-config --embed --ldflags) \ - -L${CONDA_PREFIX}/lib -Wl,-rpath,${CONDA_PREFIX}/lib \ - -o ${CONDA_PREFIX}/bin/embedded-python-shared + libpython-static) + ${CC} a.c $(python3-config --cflags) \ + $(python3-config --embed --ldflags) \ + -L${CONDA_PREFIX}/lib -Wl,-rpath,${CONDA_PREFIX}/lib \ + -o ${CONDA_PREFIX}/bin/embedded-python-static + if [[ "$target_platform" == linux-* ]]; then + if ${READELF} -d ${CONDA_PREFIX}/bin/embedded-python-static | rg '\(NEEDED\).*\[libpython[^]]*\]'; then + echo "ERROR :: Embedded python linked to shared python library. It is expected to link to the static library." + exit 1 + fi + elif [[ "$target_platform" == osx-* ]]; then + if ${OTOOL} -L ${CONDA_PREFIX}/bin/embedded-python-static | rg '^[[:space:]]+[^[:space:]]*/libpython[^[:space:]]*\.dylib[[:space:]]'; then + echo "ERROR :: Embedded python linked to shared python library. It is expected to link to the static library." + exit 1 + fi + fi + ${CONDA_PREFIX}/bin/embedded-python-static + ;; -if [[ "$target_platform" == linux-* ]]; then - if ! ${READELF} -d ${CONDA_PREFIX}/bin/embedded-python-shared | rg libpython; then - echo "ERROR :: Embedded python linked to static python library. We tried to force it to use the shared library." - fi -elif [[ "$target_platform" == osx-* ]]; then - if ! ${OTOOL} -l ${CONDA_PREFIX}/bin/embedded-python-shared | rg libpython; then - echo "ERROR :: Embedded python linked to static python library. We tried to force it to use the shared library." - fi -fi -${CONDA_PREFIX}/bin/embedded-python-shared + *) + # invalid package + exit 1 + ;; +esac set +x