From 5df573f24489ade8e20dd7d2417432ddeeb24fdf Mon Sep 17 00:00:00 2001 From: 113xiaoji <31875988+113xiaoji@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:07:49 +0800 Subject: [PATCH 1/4] build: enable aarch64 optimizations --- recipe/build_base.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/recipe/build_base.sh b/recipe/build_base.sh index 36f735b84..ad9a3347c 100755 --- a/recipe/build_base.sh +++ b/recipe/build_base.sh @@ -41,9 +41,6 @@ else fi # Since these take very long to build in our emulated ci, disable for now -if [[ ${target_platform} == linux-aarch64 ]]; then - _OPTIMIZED=no -fi if [[ ${target_platform} == linux-ppc64le ]]; then _OPTIMIZED=no fi From 8fac5bfb6f5e0c58b4c895c68e114f6ebd2ed606 Mon Sep 17 00:00:00 2001 From: 113xiaoji <31875988+113xiaoji@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:08:09 +0800 Subject: [PATCH 2/4] build: bump python build number --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 601acea26..c9d28297b 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -4,7 +4,7 @@ {% set ver2 = '.'.join(version.split('.')[0:2]) %} {% set ver2nd = ''.join(version.split('.')[0:2]) %} {% set ver3nd = ''.join(version.split('.')[0:3]) %} -{% set build_number = 0 %} +{% set build_number = 1 %} # this makes the linter happy {% set channel_targets = channel_targets or 'conda-forge main' %} From 1f6c88d4736075107e287b13ca1fc4d8f0343a04 Mon Sep 17 00:00:00 2001 From: 113xiaoji <31875988+113xiaoji@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:14:01 +0800 Subject: [PATCH 3/4] build: pin tk 8.6 for python 3.10 PPC64LE cross builds otherwise resolve Tk 9, while CPython 3.10 still expects Tcl/Tk 8.6 headers and libraries. --- recipe/conda_build_config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml index 4d63e08a7..5f798cd34 100644 --- a/recipe/conda_build_config.yaml +++ b/recipe/conda_build_config.yaml @@ -6,5 +6,7 @@ is_python_min: - no numpy: - 1.16 +tk: + - 8.6 MACOSX_SDK_VERSION: # [osx and x86_64] - 11.0 # [osx and x86_64] From 7cc8f93640e956d77753ffb46eb27efc4c7d0338 Mon Sep 17 00:00:00 2001 From: "conda-forge-webservices[bot]" <91080706+conda-forge-webservices[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:17:05 +0000 Subject: [PATCH 4/4] MNT: Re-rendered with conda-smithy 2026.9.1 and conda-forge-pinning 2026.09.02.08.53.4 --- .azure-pipelines/azure-pipelines-osx.yml | 40 +++-- .azure-pipelines/azure-pipelines-win.yml | 69 ------- .ci_support/linux_64_.yaml | 6 +- .ci_support/linux_aarch64_.yaml | 6 +- .ci_support/linux_ppc64le_.yaml | 8 +- .ci_support/osx_64_.yaml | 4 +- .ci_support/osx_arm64_.yaml | 4 +- .github/workflows/conda-build.yml | 133 +++++++++----- .scripts/build_steps.sh | 27 ++- .scripts/create_conda_build_artifacts.bat | 191 ++++++++++++++++---- .scripts/create_conda_build_artifacts.sh | 208 +++++++++++++++++----- .scripts/logging_utils.sh | 0 .scripts/run_docker_build.sh | 25 ++- .scripts/run_osx_build.sh | 4 +- .scripts/run_win_build.bat | 9 +- README.md | 61 +++++-- azure-pipelines.yml | 3 +- build-locally.py | 5 +- 18 files changed, 556 insertions(+), 247 deletions(-) mode change 100755 => 100644 .azure-pipelines/azure-pipelines-osx.yml delete mode 100755 .azure-pipelines/azure-pipelines-win.yml mode change 100644 => 100755 .scripts/logging_utils.sh diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml old mode 100755 new mode 100644 index 0f510abf6..83f4a9ab7 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -13,32 +13,38 @@ jobs: UPLOAD_PACKAGES: 'True' VMIMAGE: macOS-15 build_workspace_dir: ~/miniforge3/conda-bld + free_disk_space: skip + pagefile_size: 0 + resize_partitions: false store_build_artifacts: true tools_install_dir: ~/miniforge3 - SHORT_CONFIG: osx_64_ + CONFIG_SHORT: osx_64_ osx_arm64_: CONFIG: osx_arm64_ UPLOAD_PACKAGES: 'True' VMIMAGE: macOS-15-arm64 build_workspace_dir: ~/miniforge3/conda-bld + free_disk_space: skip + pagefile_size: 0 + resize_partitions: false store_build_artifacts: true tools_install_dir: ~/miniforge3 - SHORT_CONFIG: osx_arm64_ + CONFIG_SHORT: osx_arm64_ timeoutInMinutes: 360 variables: {} steps: # TODO: Fast finish on azure pipelines? - script: | - export MINIFORGE_HOME=$(tools_install_dir) - export CONDA_BLD_PATH=$(build_workspace_dir) export CI=azure + export CONDA_BLD_PATH=$(build_workspace_dir) + export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) + export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME + export MINIFORGE_HOME=$(tools_install_dir) + export OSX_FORCE_SDK_DOWNLOAD="1" export flow_run_id=azure_$(Build.BuildNumber).$(System.JobAttempt) export remote_url=$(Build.Repository.Uri) export sha=$(Build.SourceVersion) - export OSX_FORCE_SDK_DOWNLOAD="1" - export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME - export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) if [[ "${BUILD_REASON:-}" == "PullRequest" ]]; then export IS_PR_BUILD="True" else @@ -52,17 +58,20 @@ jobs: STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) - script: | - export MINIFORGE_HOME=$(tools_install_dir) - export CONDA_BLD_PATH=$(build_workspace_dir) + export ARTIFACT_STAGING_DIR="$(Build.ArtifactStagingDirectory)" + # Archive everything in CONDA_BLD_PATH except environments + export BLD_ARTIFACT_PREFIX=conda_pkgs export CI=azure export CI_RUN_ID=$(build.BuildNumber).$(system.JobAttempt) + export CONDA_BLD_PATH=$(build_workspace_dir) export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) - export ARTIFACT_STAGING_DIR="$(Build.ArtifactStagingDirectory)" - # Archive everything in CONDA_BLD_PATH except environments - export BLD_ARTIFACT_PREFIX=conda_artifacts + export MINIFORGE_HOME=$(tools_install_dir) if [[ "$AGENT_JOBSTATUS" == "Failed" ]]; then + # Use a distinct prefix on failure (to match GHA). + export BLD_ARTIFACT_PREFIX=conda_pkgs_failed # Archive the CONDA_BLD_PATH environments only when the job fails export ENV_ARTIFACT_PREFIX=conda_envs + export WRK_ARTIFACT_PREFIX=conda_work fi ./.scripts/create_conda_build_artifacts.sh displayName: Prepare conda build artifacts @@ -75,6 +84,13 @@ jobs: targetPath: $(BLD_ARTIFACT_PATH) artifactName: $(BLD_ARTIFACT_NAME) + - task: PublishPipelineArtifact@1 + displayName: Store conda work directoryartifacts + condition: not(eq(variables.WRK_ARTIFACT_PATH, '')) + inputs: + targetPath: $(WRK_ARTIFACT_PATH) + artifactName: $(WRK_ARTIFACT_NAME) + - task: PublishPipelineArtifact@1 displayName: Store conda build environment artifacts condition: not(eq(variables.ENV_ARTIFACT_PATH, '')) diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml deleted file mode 100755 index d364ba781..000000000 --- a/.azure-pipelines/azure-pipelines-win.yml +++ /dev/null @@ -1,69 +0,0 @@ -# This file was generated automatically from conda-smithy. To update this configuration, -# update the conda-forge.yml and/or the recipe/meta.yaml. -# -*- mode: yaml -*- - -jobs: -- job: win - pool: - vmImage: windows-2022 - strategy: - matrix: - win_64_: - CONFIG: win_64_ - UPLOAD_PACKAGES: 'True' - build_workspace_dir: D:\\bld\\ - store_build_artifacts: true - tools_install_dir: D:\Miniforge - SHORT_CONFIG: win_64_ - timeoutInMinutes: 360 - variables: - UPLOAD_TEMP: D:\\tmp - - steps: - - - script: | - call ".scripts\run_win_build.bat" - displayName: Run Windows build - env: - MINIFORGE_HOME: $(tools_install_dir) - CONDA_BLD_PATH: $(build_workspace_dir) - PYTHONUNBUFFERED: 1 - CONFIG: $(CONFIG) - CI: azure - flow_run_id: azure_$(Build.BuildNumber).$(System.JobAttempt) - remote_url: $(Build.Repository.Uri) - sha: $(Build.SourceVersion) - UPLOAD_PACKAGES: $(UPLOAD_PACKAGES) - UPLOAD_TEMP: $(UPLOAD_TEMP) - BINSTAR_TOKEN: $(BINSTAR_TOKEN) - FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN) - STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) - - - script: | - set MINIFORGE_HOME=$(tools_install_dir) - set CONDA_BLD_PATH=$(build_workspace_dir) - set CI=azure - set CI_RUN_ID=$(build.BuildNumber).$(system.JobAttempt) - set FEEDSTOCK_NAME=$(build.Repository.Name) - set ARTIFACT_STAGING_DIR=$(Build.ArtifactStagingDirectory) - set BLD_ARTIFACT_PREFIX=conda_artifacts - if "%AGENT_JOBSTATUS%" == "Failed" ( - set ENV_ARTIFACT_PREFIX=conda_envs - ) - call ".scripts\create_conda_build_artifacts.bat" - displayName: Prepare conda build artifacts - condition: and(succeededOrFailed(), eq(variables.store_build_artifacts, true)) - - - task: PublishPipelineArtifact@1 - displayName: Store conda build artifacts - condition: not(eq(variables.BLD_ARTIFACT_PATH, '')) - inputs: - targetPath: $(BLD_ARTIFACT_PATH) - artifactName: $(BLD_ARTIFACT_NAME) - - - task: PublishPipelineArtifact@1 - displayName: Store conda build environment artifacts - condition: not(eq(variables.ENV_ARTIFACT_PATH, '')) - inputs: - targetPath: $(ENV_ARTIFACT_PATH) - artifactName: $(ENV_ARTIFACT_NAME) diff --git a/.ci_support/linux_64_.yaml b/.ci_support/linux_64_.yaml index 510e4d745..053e9fce6 100644 --- a/.ci_support/linux_64_.yaml +++ b/.ci_support/linux_64_.yaml @@ -3,7 +3,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '14' +- '15' c_stdlib: - sysroot c_stdlib_version: @@ -15,9 +15,9 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '14' +- '15' docker_image: -- quay.io/condaforge/linux-anvil-x86_64:alma9 +- quay.io/condaforge/linux-anvil-x86_64:alma10 expat: - '2' liblzma_devel: diff --git a/.ci_support/linux_aarch64_.yaml b/.ci_support/linux_aarch64_.yaml index 1912c1bcb..52502bee5 100644 --- a/.ci_support/linux_aarch64_.yaml +++ b/.ci_support/linux_aarch64_.yaml @@ -3,7 +3,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '14' +- '15' c_stdlib: - sysroot c_stdlib_version: @@ -15,9 +15,9 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '14' +- '15' docker_image: -- quay.io/condaforge/linux-anvil-aarch64:alma9 +- quay.io/condaforge/linux-anvil-aarch64:alma10 expat: - '2' liblzma_devel: diff --git a/.ci_support/linux_ppc64le_.yaml b/.ci_support/linux_ppc64le_.yaml index 2c83e5ed0..0f9ddbac3 100644 --- a/.ci_support/linux_ppc64le_.yaml +++ b/.ci_support/linux_ppc64le_.yaml @@ -3,7 +3,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '14' +- '15' c_stdlib: - sysroot c_stdlib_version: @@ -15,9 +15,9 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '14' +- '15' docker_image: -- quay.io/condaforge/linux-anvil-x86_64:alma9 +- quay.io/condaforge/linux-anvil-x86_64:alma10 expat: - '2' liblzma_devel: @@ -40,6 +40,8 @@ sqlite: - '3' target_platform: - linux-ppc64le +tk: +- '8.6' zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/.ci_support/osx_64_.yaml b/.ci_support/osx_64_.yaml index 1501c3018..7d3dcad4e 100644 --- a/.ci_support/osx_64_.yaml +++ b/.ci_support/osx_64_.yaml @@ -7,7 +7,7 @@ bzip2: c_compiler: - clang c_compiler_version: -- '19' +- '21' c_stdlib: - macosx_deployment_target c_stdlib_version: @@ -19,7 +19,7 @@ channel_targets: cxx_compiler: - clangxx cxx_compiler_version: -- '19' +- '21' expat: - '2' liblzma_devel: diff --git a/.ci_support/osx_arm64_.yaml b/.ci_support/osx_arm64_.yaml index 3ea85299e..7f889f0a0 100644 --- a/.ci_support/osx_arm64_.yaml +++ b/.ci_support/osx_arm64_.yaml @@ -7,7 +7,7 @@ bzip2: c_compiler: - clang c_compiler_version: -- '19' +- '21' c_stdlib: - macosx_deployment_target c_stdlib_version: @@ -19,7 +19,7 @@ channel_targets: cxx_compiler: - clangxx cxx_compiler_version: -- '19' +- '21' expat: - '2' liblzma_devel: diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 93c3718f6..4188bff66 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -23,101 +23,129 @@ jobs: matrix: include: - CONFIG: linux_64_ + CONFIG_SHORT: linux_64_ + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma10 STORE_BUILD_ARTIFACTS: True - SHORT_CONFIG: linux_64_ UPLOAD_PACKAGES: True + build_workspace_dir: build_artifacts + docker_run_args: + free_disk_space: skip os: ubuntu + pagefile_size: 0 + resize_partitions: False runs_on: ['ubuntu-latest'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 tools_install_dir: ~/miniforge3 - build_workspace_dir: build_artifacts - docker_run_args: - CONFIG: linux_aarch64_ + CONFIG_SHORT: linux_aarch64_ + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-aarch64:alma10 STORE_BUILD_ARTIFACTS: True - SHORT_CONFIG: linux_aarch64_ UPLOAD_PACKAGES: True + build_workspace_dir: build_artifacts + docker_run_args: + free_disk_space: skip os: ubuntu + pagefile_size: 0 + resize_partitions: False runs_on: ['ubuntu-24.04-arm'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-aarch64:alma9 tools_install_dir: ~/miniforge3 - build_workspace_dir: build_artifacts - docker_run_args: - CONFIG: linux_ppc64le_ + CONFIG_SHORT: linux_ppc64le_ + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma10 STORE_BUILD_ARTIFACTS: True - SHORT_CONFIG: linux_ppc64le_ UPLOAD_PACKAGES: True + build_workspace_dir: build_artifacts + docker_run_args: + free_disk_space: skip os: ubuntu + pagefile_size: 0 + resize_partitions: False runs_on: ['ubuntu-latest'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 tools_install_dir: ~/miniforge3 - build_workspace_dir: build_artifacts + - CONFIG: win_64_ + CONFIG_SHORT: win_64_ + STORE_BUILD_ARTIFACTS: True + UPLOAD_PACKAGES: True + build_platform: win-64 + build_workspace_dir: D:\\bld\\ docker_run_args: + free_disk_space: skip + os: windows + pagefile_size: 0 + resize_partitions: False + runs_on: ['windows-2022'] + tools_install_dir: D:\Miniforge steps: - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Configure binfmt_misc + if: matrix.os == 'ubuntu' + shell: bash + run: | + if [[ "$(uname -m)" == "x86_64" ]]; then + docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes + fi - name: Build on Linux id: build-linux if: matrix.os == 'ubuntu' env: - MINIFORGE_HOME: ${{ matrix.tools_install_dir }} + CI: github_actions CONDA_BLD_PATH: ${{ matrix.build_workspace_dir }} + CONDA_FORGE_DOCKER_RUN_ARGS: ${{ matrix.docker_run_args }} CONFIG: ${{ matrix.CONFIG }} - UPLOAD_PACKAGES: ${{ matrix.UPLOAD_PACKAGES }} + MINIFORGE_HOME: ${{ matrix.tools_install_dir }} DOCKER_IMAGE: ${{ matrix.DOCKER_IMAGE }} - CI: github_actions - CONDA_FORGE_DOCKER_RUN_ARGS: "${{ matrix.docker_run_args }}" + UPLOAD_PACKAGES: ${{ matrix.UPLOAD_PACKAGES }} + RATTLER_BUILD_COLOR: always + RATTLER_BUILD_ENABLE_GITHUB_INTEGRATION: 'true' BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }} FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }} STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} shell: bash run: | - if [[ "$(uname -m)" == "x86_64" ]]; then - echo "::group::Configure binfmt_misc" - docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes - fi + export CONDA_BLD_PATH="${CONDA_BLD_PATH/#~/${HOME}}" + export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)" + export GIT_BRANCH="${GITHUB_REF_NAME}" + export MINIFORGE_HOME="${MINIFORGE_HOME/#~/${HOME}}" export flow_run_id="github_$GITHUB_RUN_ID" export remote_url="https://github.com/$GITHUB_REPOSITORY" export sha="$GITHUB_SHA" - export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)" - export GIT_BRANCH="$(basename $GITHUB_REF)" if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then export IS_PR_BUILD="True" else export IS_PR_BUILD="False" fi - export MINIFORGE_HOME="${MINIFORGE_HOME/#~/${HOME}}" - export CONDA_BLD_PATH="${CONDA_BLD_PATH/#~/${HOME}}" - echo "::endgroup::" ./.scripts/run_docker_build.sh - name: Build on macOS id: build-macos if: matrix.os == 'macos' env: - MINIFORGE_HOME: ${{ matrix.tools_install_dir }} + CI: github_actions CONDA_BLD_PATH: ${{ matrix.build_workspace_dir }} CONFIG: ${{ matrix.CONFIG }} + MINIFORGE_HOME: ${{ matrix.tools_install_dir }} UPLOAD_PACKAGES: ${{ matrix.UPLOAD_PACKAGES }} - CI: github_actions + RATTLER_BUILD_COLOR: always + RATTLER_BUILD_ENABLE_GITHUB_INTEGRATION: 'true' BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }} FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }} STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} shell: bash run: | + export CONDA_BLD_PATH="${CONDA_BLD_PATH/#~/${HOME}}" + export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)" + export GIT_BRANCH="${GITHUB_REF_NAME}" + export MINIFORGE_HOME="${MINIFORGE_HOME/#~/${HOME}}" export flow_run_id="github_$GITHUB_RUN_ID" export remote_url="https://github.com/$GITHUB_REPOSITORY" export sha="$GITHUB_SHA" - export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)" - export GIT_BRANCH="$(basename $GITHUB_REF)" if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then export IS_PR_BUILD="True" else export IS_PR_BUILD="False" fi - export MINIFORGE_HOME="${MINIFORGE_HOME/#~/${HOME}}" - export CONDA_BLD_PATH="${CONDA_BLD_PATH/#~/${HOME}}" ./.scripts/run_osx_build.sh - name: Build on windows @@ -130,12 +158,15 @@ jobs: set "sha=%GITHUB_SHA%" call ".scripts\run_win_build.bat" env: - MINIFORGE_HOME: ${{ matrix.tools_install_dir }} + BUILD_PLATFORM: ${{ matrix.build_platform }} + CI: github_actions CONDA_BLD_PATH: ${{ matrix.build_workspace_dir }} - PYTHONUNBUFFERED: 1 CONFIG: ${{ matrix.CONFIG }} - CI: github_actions + MINIFORGE_HOME: ${{ matrix.tools_install_dir }} + PYTHONUNBUFFERED: 1 UPLOAD_PACKAGES: ${{ matrix.UPLOAD_PACKAGES }} + RATTLER_BUILD_COLOR: always + RATTLER_BUILD_ENABLE_GITHUB_INTEGRATION: 'true' BINSTAR_TOKEN: ${{ secrets.BINSTAR_TOKEN }} FEEDSTOCK_TOKEN: ${{ secrets.FEEDSTOCK_TOKEN }} STAGING_BINSTAR_TOKEN: ${{ secrets.STAGING_BINSTAR_TOKEN }} @@ -169,25 +200,27 @@ jobs: # we do not want to trigger artefact creation if the build was cancelled if: ${{ always() && steps.determine-status.outputs.status != 'cancelled' && matrix.STORE_BUILD_ARTIFACTS }} env: - CONDA_BLD_PATH: ${{ matrix.build_workspace_dir }} CI: github_actions + CONDA_BLD_PATH: ${{ matrix.build_workspace_dir }} CONFIG: ${{ matrix.CONFIG }} - SHORT_CONFIG: ${{ matrix.SHORT_CONFIG }} + CONFIG_SHORT: ${{ matrix.CONFIG_SHORT }} JOB_STATUS: ${{ steps.determine-status.outputs.status }} + MINIFORGE_HOME: ${{ matrix.tools_install_dir }} OS: ${{ matrix.os }} run: | - export MINIFORGE_HOME="${MINIFORGE_HOME/#~/${HOME}}" - export CONDA_BLD_PATH="${CONDA_BLD_PATH/#~/${HOME}}" + export ARTIFACT_STAGING_DIR="$GITHUB_WORKSPACE" export CI_RUN_ID=$GITHUB_RUN_ID + export CONDA_BLD_PATH="${CONDA_BLD_PATH/#~/${HOME}}" export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)" - export ARTIFACT_STAGING_DIR="$GITHUB_WORKSPACE" + export MINIFORGE_HOME="${MINIFORGE_HOME/#~/${HOME}}" # Archive everything in CONDA_BLD_PATH except environments # Archive the CONDA_BLD_PATH environments only when the job fails # Use different prefix for successful and failed build artifacts # so random failures don't prevent rebuilds from creating artifacts. if [ $JOB_STATUS == "failure" ]; then - export BLD_ARTIFACT_PREFIX="conda_artifacts" + export BLD_ARTIFACT_PREFIX="conda_pkgs_failed" export ENV_ARTIFACT_PREFIX="conda_envs" + export WRK_ARTIFACT_PREFIX="conda_work" else export BLD_ARTIFACT_PREFIX="conda_pkgs" fi @@ -198,20 +231,32 @@ jobs: fi - name: Store conda build artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - if: ${{ always() && steps.prepare-artifacts.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ always() && steps.prepare-artifacts.outcome == 'success' && steps.prepare-artifacts.outputs.BLD_ARTIFACT_PATH != '' }} with: name: ${{ steps.prepare-artifacts.outputs.BLD_ARTIFACT_NAME }} path: ${{ steps.prepare-artifacts.outputs.BLD_ARTIFACT_PATH }} retention-days: 14 + archive: false + continue-on-error: true + + - name: Store conda work directory artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ always() && steps.prepare-artifacts.outcome == 'success' && steps.prepare-artifacts.outputs.WRK_ARTIFACT_PATH != '' }} + with: + name: ${{ steps.prepare-artifacts.outputs.WRK_ARTIFACT_NAME }} + path: ${{ steps.prepare-artifacts.outputs.WRK_ARTIFACT_PATH }} + retention-days: 14 + archive: false continue-on-error: true - name: Store conda build environment artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 # only relevant if build failed, see above - if: ${{ always() && steps.determine-status.outputs.status == 'failure' && steps.prepare-artifacts.outcome == 'success' }} + if: ${{ always() && steps.prepare-artifacts.outcome == 'success' && steps.prepare-artifacts.outputs.ENV_ARTIFACT_PATH != '' }} with: name: ${{ steps.prepare-artifacts.outputs.ENV_ARTIFACT_NAME }} path: ${{ steps.prepare-artifacts.outputs.ENV_ARTIFACT_PATH }} retention-days: 14 + archive: false continue-on-error: true \ No newline at end of file diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index c83e212f4..484acddad 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -61,18 +61,33 @@ if [[ -f "${FEEDSTOCK_ROOT}/LICENSE.txt" ]]; then fi if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then - if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then - EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}" - fi - conda debug "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ + # differences between conda-build vs. rattler-build + # - 1 step (conda debug + manually open shell) vs. 2 step (rb debug {setup, shell}) + # - recipe is positional vs. --recipe "${RECIPE_ROOT}" + # - --output-id vs. --output-name + # - --clobber-file vs. none + # - none vs. --target-platform + CONDA_SUBDIR="${BUILD_PLATFORM}" conda debug \ + "${RECIPE_ROOT}" \ + -m "${CI_SUPPORT}/${CONFIG}.yaml" \ ${EXTRA_CB_OPTIONS:-} \ + ${BUILD_OUTPUT_ID:+--output-id "${BUILD_OUTPUT_ID}"} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" # Drop into an interactive shell /bin/bash else - conda-build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ - --suppress-variables ${EXTRA_CB_OPTIONS:-} \ + # differences between conda-build vs. rattler-build + # - recipe is positional vs. --recipe "${RECIPE_ROOT}" + # - --suppress-variables vs. none + # - --clobber-file vs. none + # - none vs. --target-platform + # - --extra-meta a=b c=d vs. --extra-meta a=b --extra-meta c=d + CONDA_SUBDIR="${BUILD_PLATFORM}" conda-build \ + "${RECIPE_ROOT}" \ + -m "${CI_SUPPORT}/${CONFIG}.yaml" \ + ${EXTRA_CB_OPTIONS:-} \ + --suppress-variables \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ --extra-meta flow_run_id="${flow_run_id:-}" remote_url="${remote_url:-}" sha="${sha:-}" ( startgroup "Inspecting artifacts" ) 2> /dev/null diff --git a/.scripts/create_conda_build_artifacts.bat b/.scripts/create_conda_build_artifacts.bat index 6120712cc..487539f26 100755 --- a/.scripts/create_conda_build_artifacts.bat +++ b/.scripts/create_conda_build_artifacts.bat @@ -1,16 +1,19 @@ +@echo off setlocal enableextensions enabledelayedexpansion rem INPUTS (environment variables that need to be set before calling this script): rem rem CI (azure/github_actions/UNSET) rem CI_RUN_ID (unique identifier for the CI job run) -rem FEEDSTOCK_NAME -rem CONFIG (build matrix configuration string) -rem SHORT_CONFIG (uniquely-shortened configuration string) rem CONDA_BLD_PATH (path to the conda-bld directory) +rem CONFIG (build matrix configuration string) +rem CONFIG_SHORT (uniquely-shortened configuration string) +rem FEEDSTOCK_NAME +rem Optional: rem ARTIFACT_STAGING_DIR (use working directory if unset) rem BLD_ARTIFACT_PREFIX (prefix for the conda build artifact name, skip if unset) rem ENV_ARTIFACT_PREFIX (prefix for the conda build environments artifact name, skip if unset) +rem WRK_ARTIFACT_PREFIX (prefix for the conda work directory artifact name, skip if unset) rem OUTPUTS rem @@ -18,6 +21,10 @@ rem BLD_ARTIFACT_NAME rem BLD_ARTIFACT_PATH rem ENV_ARTIFACT_NAME rem ENV_ARTIFACT_PATH +rem WRK_ARTIFACT_NAME +rem WRK_ARTIFACT_PATH +call "%MINIFORGE_HOME%\Scripts\activate.bat" +if !errorlevel! neq 0 exit /b !errorlevel! rem Check that the conda-build directory exists if not exist %CONDA_BLD_PATH% ( @@ -33,45 +40,167 @@ if not defined ARTIFACT_STAGING_DIR ( rem Set a unique ID for the artifact(s), specialized for this particular job run set ARTIFACT_UNIQUE_ID=%CI_RUN_ID%_%CONFIG% if not "%ARTIFACT_UNIQUE_ID%" == "%ARTIFACT_UNIQUE_ID:~0,80%" ( - set ARTIFACT_UNIQUE_ID=%CI_RUN_ID%_%SHORT_CONFIG% + set ARTIFACT_UNIQUE_ID=%CI_RUN_ID%_%CONFIG_SHORT% ) -rem Make the build artifact zip +rem --use-compress-prog hangs on Azure, but we can use bsdtar unconditionally. +set "ZSTD=--zstd --options=zstd:compression-level=12,zstd:threads=0" + +rem Mirror the logic from create_conda_build_artifacts.sh.tmpl. +rem Note that the paths to tar must use forward slashes. + +cd "%CONDA_BLD_PATH%" +if errorlevel 1 exit 1 + +set ARTIFACT_PATHS= +set ENVIRONMENT_PATHS= +set "EXCLUDE_FROM_WORK=--exclude=.git --exclude=./pkg_cache --exclude=./src_cache" + +for /d %%a in (linux-* osx-* win-* broken noarch) do ( + if exist %%a ( + set "ARTIFACT_PATHS=!ARTIFACT_PATHS! %%a" + set "EXCLUDE_FROM_WORK=!EXCLUDE_FROM_WORK! --exclude=%%a" + ) +) +for %%a in (channeldata.json index.html) do ( + if exist %%a ( + set "ARTIFACT_PATHS=!ARTIFACT_PATHS! %%a" + set "EXCLUDE_FROM_WORK=!EXCLUDE_FROM_WORK! --exclude=%%a" + ) +) +for /d %%a in (. bld test) do ( + if exist %%a ( + pushd %%a + if errorlevel 1 exit 1 + + for /d %%b in (*_*) do ( + if exist %%a\%%b\pip_cache ( + set "EXCLUDE_FROM_WORK=!EXCLUDE_FROM_WORK! --exclude=%%a/%%b/pip_cache" + ) + + pushd %%b + if errorlevel 1 exit 1 + + for /d %%c in (*_env* *_prefix_moved_*) do ( + set "ENVIRONMENT_PATHS=!ENVIRONMENT_PATHS! %%a/%%b/%%c" + set "EXCLUDE_FROM_WORK=!EXCLUDE_FROM_WORK! --exclude=%%a/%%b/%%c" + ) + + popd + if errorlevel 1 exit 1 + ) + + popd + if errorlevel 1 exit 1 + ) +) + +rem Make the build artifact archive if defined BLD_ARTIFACT_PREFIX ( - set BLD_ARTIFACT_NAME=%BLD_ARTIFACT_PREFIX%_%ARTIFACT_UNIQUE_ID% - echo BLD_ARTIFACT_NAME: !BLD_ARTIFACT_NAME! + if not "%ARTIFACT_PATHS%" == "" ( + echo Creating build artifact archive ... + set BLD_ARTIFACT_NAME=%BLD_ARTIFACT_PREFIX%_%ARTIFACT_UNIQUE_ID% + + set "BLD_ARTIFACT_PATH=%ARTIFACT_STAGING_DIR%\%FEEDSTOCK_NAME%_%BLD_ARTIFACT_PREFIX%_%ARCHIVE_UNIQUE_ID%.tar.zst" + bsdtar -c -f "!BLD_ARTIFACT_PATH!" %ZSTD% %ARTIFACT_PATHS% + if errorlevel 1 ( + if exist "!BLD_ARTIFACT_PATH!" ( + move "!BLD_ARTIFACT_PATH!" "!BLD_ARTIFACT_PATH:.tar.zst=-broken.tar.zst!" + set "BLD_ARTIFACT_PATH=!BLD_ARTIFACT_PATH:.tar.zst=-broken.tar.zst!" + ) + ) + + if exist "!BLD_ARTIFACT_PATH!" ( + echo Archive created: + dir "!BLD_ARTIFACT_PATH!" - set "BLD_ARTIFACT_PATH=%ARTIFACT_STAGING_DIR%\%FEEDSTOCK_NAME%_%BLD_ARTIFACT_PREFIX%_%ARCHIVE_UNIQUE_ID%.zip" - 7z a "!BLD_ARTIFACT_PATH!" "%CONDA_BLD_PATH%" -xr^^!.git/ -xr^^!_*_env*/ -xr^^!*_cache/ -bb - if errorlevel 1 exit 1 - echo BLD_ARTIFACT_PATH: !BLD_ARTIFACT_PATH! + if "%CI%" == "azure" ( + echo ##vso[task.setVariable variable=BLD_ARTIFACT_NAME]!BLD_ARTIFACT_NAME! + echo ##vso[task.setVariable variable=BLD_ARTIFACT_PATH]!BLD_ARTIFACT_PATH! + ) + if "%CI%" == "github_actions" ( + echo BLD_ARTIFACT_NAME=!BLD_ARTIFACT_NAME!>> !GITHUB_OUTPUT! + echo BLD_ARTIFACT_PATH=!BLD_ARTIFACT_PATH!>> !GITHUB_OUTPUT! + ) + ) else ( + echo No archive created + ) + ) else ( + echo Skipping work directory artifact archive because of no files + ) +) else ( + echo Skipping build artifact archive +) +echo. - if "%CI%" == "azure" ( - echo ##vso[task.setVariable variable=BLD_ARTIFACT_NAME]!BLD_ARTIFACT_NAME! - echo ##vso[task.setVariable variable=BLD_ARTIFACT_PATH]!BLD_ARTIFACT_PATH! +rem Make the work directory artifact archive +if defined WRK_ARTIFACT_PREFIX ( + echo Creating work directory archive ... + set WRK_ARTIFACT_NAME=!WRK_ARTIFACT_PREFIX!_%ARTIFACT_UNIQUE_ID% + + set "WRK_ARTIFACT_PATH=%ARTIFACT_STAGING_DIR%\%FEEDSTOCK_NAME%_%WRK_ARTIFACT_PREFIX%_%ARCHIVE_UNIQUE_ID%.tar.zst" + bsdtar -c -f "!WRK_ARTIFACT_PATH!" %ZSTD% %EXCLUDE_FROM_WORK% . + if errorlevel 1 ( + if exist "!WRK_ARTIFACT_PATH!" ( + move "!WRK_ARTIFACT_PATH!" "!WRK_ARTIFACT_PATH:.tar.zst=-broken.tar.zst!" + set "WRK_ARTIFACT_PATH=!WRK_ARTIFACT_PATH:.tar.zst=-broken.tar.zst!" + ) ) - if "%CI%" == "github_actions" ( - echo BLD_ARTIFACT_NAME=!BLD_ARTIFACT_NAME!>> !GITHUB_OUTPUT! - echo BLD_ARTIFACT_PATH=!BLD_ARTIFACT_PATH!>> !GITHUB_OUTPUT! + + if exist "!WRK_ARTIFACT_PATH!" ( + echo Archive created: + dir "!WRK_ARTIFACT_PATH!" + + if "%CI%" == "azure" ( + echo ##vso[task.setVariable variable=WRK_ARTIFACT_NAME]!WRK_ARTIFACT_NAME! + echo ##vso[task.setVariable variable=WRK_ARTIFACT_PATH]!WRK_ARTIFACT_PATH! + ) + if "%CI%" == "github_actions" ( + echo WRK_ARTIFACT_NAME=!WRK_ARTIFACT_NAME!>> !GITHUB_OUTPUT! + echo WRK_ARTIFACT_PATH=!WRK_ARTIFACT_PATH!>> !GITHUB_OUTPUT! + ) + ) else ( + echo No archive created ) +) else ( + echo Skipping work directory artifact archive ) +echo. -rem Make the environments artifact zip +rem Make the environment artifact archive if defined ENV_ARTIFACT_PREFIX ( - set ENV_ARTIFACT_NAME=!ENV_ARTIFACT_PREFIX!_%ARTIFACT_UNIQUE_ID% - echo ENV_ARTIFACT_NAME: !ENV_ARTIFACT_NAME! + if not "%ENVIRONMENT_PATHS%" == "" ( + echo Creating build environment artifact archive ... + set ENV_ARTIFACT_NAME=!ENV_ARTIFACT_PREFIX!_%ARTIFACT_UNIQUE_ID% - set "ENV_ARTIFACT_PATH=%ARTIFACT_STAGING_DIR%\%FEEDSTOCK_NAME%_%ENV_ARTIFACT_PREFIX%_%ARCHIVE_UNIQUE_ID%.zip" - 7z a "!ENV_ARTIFACT_PATH!" -r "%CONDA_BLD_PATH%"/_*_env*/ -bb - if errorlevel 1 exit 1 - echo ENV_ARTIFACT_PATH: !ENV_ARTIFACT_PATH! + set "ENV_ARTIFACT_PATH=%ARTIFACT_STAGING_DIR%\%FEEDSTOCK_NAME%_%ENV_ARTIFACT_PREFIX%_%ARCHIVE_UNIQUE_ID%.tar.zst" + bsdtar -c -f "!ENV_ARTIFACT_PATH!" %ZSTD% %ENVIRONMENT_PATHS% + if errorlevel 1 ( + if exist "!ENV_ARTIFACT_PATH!" ( + move "!ENV_ARTIFACT_PATH!" "!ENV_ARTIFACT_PATH:.tar.zst=-broken.tar.zst!" + set "ENV_ARTIFACT_PATH=!ENV_ARTIFACT_PATH:.tar.zst=-broken.tar.zst!" + ) + ) - if "%CI%" == "azure" ( - echo ##vso[task.setVariable variable=ENV_ARTIFACT_NAME]!ENV_ARTIFACT_NAME! - echo ##vso[task.setVariable variable=ENV_ARTIFACT_PATH]!ENV_ARTIFACT_PATH! - ) - if "%CI%" == "github_actions" ( - echo ENV_ARTIFACT_NAME=!ENV_ARTIFACT_NAME!>> !GITHUB_OUTPUT! - echo ENV_ARTIFACT_PATH=!ENV_ARTIFACT_PATH!>> !GITHUB_OUTPUT! + if exist "!ENV_ARTIFACT_PATH!" ( + echo Archive created: + dir "!ENV_ARTIFACT_PATH!" + + if "%CI%" == "azure" ( + echo ##vso[task.setVariable variable=ENV_ARTIFACT_NAME]!ENV_ARTIFACT_NAME! + echo ##vso[task.setVariable variable=ENV_ARTIFACT_PATH]!ENV_ARTIFACT_PATH! + ) + if "%CI%" == "github_actions" ( + echo ENV_ARTIFACT_NAME=!ENV_ARTIFACT_NAME!>> !GITHUB_OUTPUT! + echo ENV_ARTIFACT_PATH=!ENV_ARTIFACT_PATH!>> !GITHUB_OUTPUT! + ) + ) else ( + echo No archive created + ) + ) else ( + echo Skipping environment artifact archive because of no files ) +) else ( + echo Skipping environment artifact archive ) +echo. diff --git a/.scripts/create_conda_build_artifacts.sh b/.scripts/create_conda_build_artifacts.sh index 1e3acc78a..9e4f4e2d8 100755 --- a/.scripts/create_conda_build_artifacts.sh +++ b/.scripts/create_conda_build_artifacts.sh @@ -4,13 +4,15 @@ # # CI (azure/github_actions/UNSET) # CI_RUN_ID (unique identifier for the CI job run) -# FEEDSTOCK_NAME -# CONFIG (build matrix configuration string) -# SHORT_CONFIG (uniquely-shortened configuration string) # CONDA_BLD_PATH (path to the conda-bld directory) +# CONFIG (build matrix configuration string) +# CONFIG_SHORT (uniquely-shortened configuration string) +# FEEDSTOCK_NAME +# Optional: # ARTIFACT_STAGING_DIR (use working directory if unset) # BLD_ARTIFACT_PREFIX (prefix for the conda build artifact name, skip if unset) # ENV_ARTIFACT_PREFIX (prefix for the conda build environments artifact name, skip if unset) +# WRK_ARTIFACT_PREFIX (prefix for the conda build work artifact name, skip if unset) # OUTPUTS # @@ -18,6 +20,8 @@ # BLD_ARTIFACT_PATH # ENV_ARTIFACT_NAME # ENV_ARTIFACT_PATH +# WRK_ARTIFACT_NAME +# WRK_ARTIFACT_PATH source .scripts/logging_utils.sh @@ -25,6 +29,12 @@ source .scripts/logging_utils.sh # and that might end up inserting extraneous quotation marks in output variables set -e +# mangle_homebrew hides zstd on GHA macos 15 runners, so use conda-forge tools +if [[ -d ${MINIFORGE_HOME} ]]; then + . "${MINIFORGE_HOME}/etc/profile.d/conda.sh" + conda activate +fi + # Check that the conda-build directory exists if [ ! -d "$CONDA_BLD_PATH" ]; then echo "conda-build directory does not exist" @@ -49,65 +59,171 @@ fi # Set a unique ID for the artifact(s), specialized for this particular job run ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${CONFIG}" if [[ ${#ARTIFACT_UNIQUE_ID} -gt 80 ]]; then - ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${SHORT_CONFIG}" + ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${CONFIG_SHORT}" fi echo "ARTIFACT_UNIQUE_ID: $ARTIFACT_UNIQUE_ID" # Set a descriptive ID for the archive(s), specialized for this particular job run ARCHIVE_UNIQUE_ID="${CI_RUN_ID}_${CONFIG}" -# Make the build artifact zip -if [[ ! -z "$BLD_ARTIFACT_PREFIX" ]]; then - export BLD_ARTIFACT_NAME="${BLD_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" - export BLD_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${BLD_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.zip" +pushd "${CONDA_BLD_PATH}" - ( startgroup "Archive conda build directory" ) 2> /dev/null +# --use-compress-prog= lets us pass the options to GNU tar and libarchive tar +# -T0 uses all cores +# 12 gives reasonable compression while remaining fast +ZSTD="--use-compress-prog=zstd -T0 -12" - # Try 7z and fall back to zip if it fails (for cross-platform use) - if ! 7z a "$BLD_ARTIFACT_PATH" "$CONDA_BLD_PATH" '-xr!.git/' '-xr!_*_env*/' '-xr!*_cache/' -bb; then - pushd "$CONDA_BLD_PATH" - zip -r -y -T "$BLD_ARTIFACT_PATH" . -x '*.git/*' '*_*_env*/*' '*_cache/*' - popd +# Pattern matching can be quite overzealous, so rather than passing wildcards, +# we want to expand them into actual paths first. +# +# The split is: +# - artifacts: all packages and related repository files +# - environments: build, host and test environments +# - work directory: everything else (modulo excludes) + +shopt -s nullglob +ARTIFACT_PATHS=( + # all possible platform directories + linux-* + osx-* + win-* + + # appending '*' here to have nullglob skip these directories and files + # conveniently if they don't exist + broken* + noarch* + channeldata.json* + index.html* +) +ENVIRONMENT_PATHS=( + # note: always include "./", so that only top-level paths match + # conda-build + ./*_*/*_env* + ./*_*/*_prefix_moved_* + + # rattler-build + ./bld/*_*/*_env* + ./test/test_*/test_env +) + +EXCLUDE_FROM_WORK=( + .git + + # caches + ./pkg_cache + ./src_cache + ./*_*/pip_cache + + # artifacts and environments that go into separate archives + "${ARTIFACT_PATHS[@]}" + "${ENVIRONMENT_PATHS[@]}" +) + +# Make the build artifact archive +if [[ ! -z ${BLD_ARTIFACT_PREFIX} && -n ${ARTIFACT_PATHS} ]]; then + echo "Creating build artifact archive ..." + export BLD_ARTIFACT_NAME="${BLD_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" + export BLD_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${BLD_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.tar.zst" + + # All our CI services have either GNU tar or bsdtar, and zstd. + # Keep the command compatible with both! + if ! tar -c -f "${BLD_ARTIFACT_PATH}" "${ZSTD}" "${ARTIFACT_PATHS[@]}" && + [[ -s ${BLD_ARTIFACT_PATH} ]] + then + # If tar failed but produced a (partial?) file, upload it as "broken". + mv -v "${BLD_ARTIFACT_PATH}" "${BLD_ARTIFACT_PATH/%.tar.zst/-broken&}" + BLD_ARTIFACT_NAME+=-broken + BLD_ARTIFACT_PATH=${BLD_ARTIFACT_PATH/%.tar.zst/-broken&} fi - ( endgroup "Archive conda build directory" ) 2> /dev/null - - echo "BLD_ARTIFACT_NAME: $BLD_ARTIFACT_NAME" - echo "BLD_ARTIFACT_PATH: $BLD_ARTIFACT_PATH" + if [[ -s ${BLD_ARTIFACT_PATH} ]]; then + echo "Archive created:" + ls -l -h "${BLD_ARTIFACT_PATH}" + + if [[ "$CI" == "azure" ]]; then + echo "##vso[task.setVariable variable=BLD_ARTIFACT_NAME]$BLD_ARTIFACT_NAME" + echo "##vso[task.setVariable variable=BLD_ARTIFACT_PATH]$BLD_ARTIFACT_PATH" + elif [[ "$CI" == "github_actions" ]]; then + echo "BLD_ARTIFACT_NAME=$BLD_ARTIFACT_NAME" >> $GITHUB_OUTPUT + echo "BLD_ARTIFACT_PATH=$BLD_ARTIFACT_PATH" >> $GITHUB_OUTPUT + fi + else + echo "No archive created" + fi +else + echo "Skipping build artifact archive" +fi +echo + +# Make the workdir artifact archive +if [[ ! -z ${WRK_ARTIFACT_PREFIX} ]]; then + echo "Creating work directory archive ..." + export WRK_ARTIFACT_NAME="${WRK_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" + export WRK_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${WRK_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.tar.zst" + + # All our CI services have either GNU tar or bsdtar, and zstd. + # Keep the command compatible with both! + if ! tar -c -f "${WRK_ARTIFACT_PATH}" "${ZSTD}" \ + "${EXCLUDE_FROM_WORK[@]/#/--exclude=}" . && + [[ -s ${WRK_ARTIFACT_PATH} ]] + then + # If tar failed but produced a (partial?) file, upload it as "broken". + mv -v "${WRK_ARTIFACT_PATH}" "${WRK_ARTIFACT_PATH/%.tar.zst/-broken&}" + WRK_ARTIFACT_NAME+=-broken + WRK_ARTIFACT_PATH=${WRK_ARTIFACT_PATH/%.tar.zst/-broken&} + fi - if [[ "$CI" == "azure" ]]; then - echo "##vso[task.setVariable variable=BLD_ARTIFACT_NAME]$BLD_ARTIFACT_NAME" - echo "##vso[task.setVariable variable=BLD_ARTIFACT_PATH]$BLD_ARTIFACT_PATH" - elif [[ "$CI" == "github_actions" ]]; then - echo "BLD_ARTIFACT_NAME=$BLD_ARTIFACT_NAME" >> $GITHUB_OUTPUT - echo "BLD_ARTIFACT_PATH=$BLD_ARTIFACT_PATH" >> $GITHUB_OUTPUT + if [[ -s ${WRK_ARTIFACT_PATH} ]]; then + echo "Archive created:" + ls -l -h "${WRK_ARTIFACT_PATH}" + + if [[ "$CI" == "azure" ]]; then + echo "##vso[task.setVariable variable=WRK_ARTIFACT_NAME]$WRK_ARTIFACT_NAME" + echo "##vso[task.setVariable variable=WRK_ARTIFACT_PATH]$WRK_ARTIFACT_PATH" + elif [[ "$CI" == "github_actions" ]]; then + echo "WRK_ARTIFACT_NAME=$WRK_ARTIFACT_NAME" >> $GITHUB_OUTPUT + echo "WRK_ARTIFACT_PATH=$WRK_ARTIFACT_PATH" >> $GITHUB_OUTPUT + fi + else + echo "No archive created" fi +else + echo "Skipping work directory artifact archive" fi +echo -# Make the environments artifact zip -if [[ ! -z "$ENV_ARTIFACT_PREFIX" ]]; then +# Make the environments artifact archive +if [[ ! -z ${ENV_ARTIFACT_PREFIX} && -n ${ENVIRONMENT_PATHS[@]} ]]; then + echo "Creating build environment artifact archive ..." export ENV_ARTIFACT_NAME="${ENV_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" - export ENV_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${ENV_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.zip" - - ( startgroup "Archive conda build environments" ) 2> /dev/null - - # Try 7z and fall back to zip if it fails (for cross-platform use) - if ! 7z a "$ENV_ARTIFACT_PATH" -r "$CONDA_BLD_PATH"/'_*_env*/' -bb; then - pushd "$CONDA_BLD_PATH" - zip -r -y -T "$ENV_ARTIFACT_PATH" . -i '*_*_env*/*' - popd + export ENV_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${ENV_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.tar.zst" + + if ! tar -c -f "${ENV_ARTIFACT_PATH}" "${ZSTD}" "${ENVIRONMENT_PATHS[@]}" && + [[ -s ${ENV_ARTIFACT_PATH} ]] + then + # If tar failed but produced a (partial?) file, upload it as "broken". + mv -v "${ENV_ARTIFACT_PATH}" "${ENV_ARTIFACT_PATH/%.tar.zst/-broken&}" + ENV_ARTIFACT_NAME+=-broken + ENV_ARTIFACT_PATH=${ENV_ARTIFACT_PATH/%.tar.zst/-broken&} fi - ( endgroup "Archive conda build environments" ) 2> /dev/null - - echo "ENV_ARTIFACT_NAME: $ENV_ARTIFACT_NAME" - echo "ENV_ARTIFACT_PATH: $ENV_ARTIFACT_PATH" - - if [[ "$CI" == "azure" ]]; then - echo "##vso[task.setVariable variable=ENV_ARTIFACT_NAME]$ENV_ARTIFACT_NAME" - echo "##vso[task.setVariable variable=ENV_ARTIFACT_PATH]$ENV_ARTIFACT_PATH" - elif [[ "$CI" == "github_actions" ]]; then - echo "ENV_ARTIFACT_NAME=$ENV_ARTIFACT_NAME" >> $GITHUB_OUTPUT - echo "ENV_ARTIFACT_PATH=$ENV_ARTIFACT_PATH" >> $GITHUB_OUTPUT + if [[ -s ${ENV_ARTIFACT_PATH} ]]; then + echo "Archive created:" + ls -l -h "${ENV_ARTIFACT_PATH}" + + if [[ "$CI" == "azure" ]]; then + echo "##vso[task.setVariable variable=ENV_ARTIFACT_NAME]$ENV_ARTIFACT_NAME" + echo "##vso[task.setVariable variable=ENV_ARTIFACT_PATH]$ENV_ARTIFACT_PATH" + elif [[ "$CI" == "github_actions" ]]; then + echo "ENV_ARTIFACT_NAME=$ENV_ARTIFACT_NAME" >> $GITHUB_OUTPUT + echo "ENV_ARTIFACT_PATH=$ENV_ARTIFACT_PATH" >> $GITHUB_OUTPUT + fi + else + echo "No archive created" fi +else + echo "Skipping environment artifact archive" fi +echo + +popd diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh old mode 100644 new mode 100755 diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 10d6a2af9..17d983f11 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -96,6 +96,14 @@ if [ "${DOCKER_EXECUTABLE}" = "podman" ]; then fi fi +# When running on GitHub Actions, mount the step summary file into the container +# and point GITHUB_STEP_SUMMARY at it so that rattler-build's GitHub integration +# can write its summary. GitHub writes the file out to the job summary after the +# step finishes. +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} -v ${GITHUB_STEP_SUMMARY}:/home/conda/github_step_summary:rw${VOLUME_SUFFIX},delegated -e GITHUB_STEP_SUMMARY=/home/conda/github_step_summary" +fi + ( endgroup "Configure Docker" ) 2> /dev/null ( startgroup "Start Docker" ) 2> /dev/null @@ -107,17 +115,20 @@ ${DOCKER_EXECUTABLE} pull "${DOCKER_IMAGE}" ${DOCKER_EXECUTABLE} run ${DOCKER_RUN_ARGS} \ -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw${VOLUME_SUFFIX},delegated \ -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw${VOLUME_SUFFIX},delegated \ + -e BUILD_OUTPUT_ID \ + -e BUILD_WITH_CONDA_DEBUG \ + -e CI \ -e CONFIG \ + -e CPU_COUNT \ + -e FEEDSTOCK_NAME \ + -e GIT_BRANCH \ + -e GITHUB_ACTIONS \ -e HOST_USER_ID \ - -e UPLOAD_PACKAGES \ -e IS_PR_BUILD \ - -e GIT_BRANCH \ + -e RATTLER_BUILD_COLOR \ + -e RATTLER_BUILD_ENABLE_GITHUB_INTEGRATION \ -e UPLOAD_ON_BRANCH \ - -e CI \ - -e FEEDSTOCK_NAME \ - -e CPU_COUNT \ - -e BUILD_WITH_CONDA_DEBUG \ - -e BUILD_OUTPUT_ID \ + -e UPLOAD_PACKAGES \ -e flow_run_id \ -e remote_url \ -e sha \ diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index b0eda55cd..9d7c39125 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -100,7 +100,7 @@ if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}" fi - conda debug ./recipe -m ./.ci_support/${CONFIG}.yaml \ + CONDA_SUBDIR="${BUILD_PLATFORM}" conda debug ./recipe -m ./.ci_support/${CONFIG}.yaml \ ${EXTRA_CB_OPTIONS:-} \ --clobber-file ./.ci_support/clobber_${CONFIG}.yaml @@ -112,7 +112,7 @@ else EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" fi - conda-build ./recipe -m ./.ci_support/${CONFIG}.yaml \ + CONDA_SUBDIR="${BUILD_PLATFORM}" conda-build ./recipe -m ./.ci_support/${CONFIG}.yaml \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \ --extra-meta flow_run_id="$flow_run_id" remote_url="$remote_url" sha="$sha" diff --git a/.scripts/run_win_build.bat b/.scripts/run_win_build.bat index c07d98e61..6a6309e57 100755 --- a/.scripts/run_win_build.bat +++ b/.scripts/run_win_build.bat @@ -42,7 +42,8 @@ call :start_group "Configuring conda" :: Activate the base conda environment echo Activating environment -call "%MINIFORGE_HOME%\Scripts\activate.bat" +set "CONDA_EXE=%MINIFORGE_HOME%\Scripts\conda.exe" +call "%MINIFORGE_HOME%\condabin\conda.bat" activate "%MINIFORGE_HOME%" :: Configure the solver set "CONDA_SOLVER=libmamba" if !errorlevel! neq 0 exit /b !errorlevel! @@ -76,8 +77,12 @@ call :end_group :: Build the recipe echo Building recipe +set "_OLD_CONDA_SUBDIR=%CONDA_SUBDIR%" +set "CONDA_SUBDIR=%BUILD_PLATFORM%" conda-build.exe "recipe" -m .ci_support\%CONFIG%.yaml --suppress-variables %EXTRA_CB_OPTIONS% if !errorlevel! neq 0 exit /b !errorlevel! +set "_OLD_CONDA_SUBDIR=" +set "CONDA_SUBDIR=%_OLD_CONDA_SUBDIR%" call :start_group "Inspecting artifacts" :: inspect_artifacts was only added in conda-forge-ci-setup 4.9.4 @@ -87,7 +92,7 @@ call :end_group :: Prepare some environment variables for the upload step if /i "%CI%" == "github_actions" ( set "FEEDSTOCK_NAME=%GITHUB_REPOSITORY:*/=%" - set "GIT_BRANCH=%GITHUB_REF:refs/heads/=%" + set "GIT_BRANCH=%GITHUB_REF_NAME%" if /i "%GITHUB_EVENT_NAME%" == "pull_request" ( set "IS_PR_BUILD=True" ) else ( diff --git a/README.md b/README.md index 2779a3a71..c06622526 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,6 @@ Current build status variant - - win_64 - - - variant - - @@ -94,31 +87,73 @@ conda config --add channels conda-forge conda config --set channel_priority strict ``` -Once the `conda-forge` channel has been enabled, `cpython, libpython-static, python, python-gil` can be installed with `conda`: +How to use +---------- + +
+With conda ``` conda install cpython libpython-static python python-gil ``` -or with `mamba`: +
+ +
+With mamba ``` mamba install cpython libpython-static python python-gil ``` -It is possible to list all of the versions of `cpython` available on your platform with `conda`: +
+ +
+With pixi + +``` +# for adding to your local project +pixi add cpython libpython-static python python-gil +# for installing globally +pixi global install cpython libpython-static python python-gil +``` + +
+ +Search package versions +----------------------- + +It is possible to list all of the versions of `cpython` available on your platform: + +
+With conda ``` conda search cpython --channel conda-forge ``` -or with `mamba`: +
+ +
+With mamba ``` mamba search cpython --channel conda-forge ``` -Alternatively, `mamba repoquery` may provide more information: +
+ +
+With pixi + +``` +pixi search cpython --channel conda-forge +``` + +
+ +
+With mamba repoquery, which may provide more information ``` # Search all versions available on your platform: @@ -131,6 +166,8 @@ mamba repoquery whoneeds cpython --channel conda-forge mamba repoquery depends cpython --channel conda-forge ``` +
+ About conda-forge ================= diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 03594e446..7e0f633df 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -28,5 +28,4 @@ stages: condition: and(succeeded(), eq(dependencies.Check.outputs['Skip.result.start_main'], 'true')) dependsOn: Check jobs: - - template: ./.azure-pipelines/azure-pipelines-osx.yml - - template: ./.azure-pipelines/azure-pipelines-win.yml \ No newline at end of file + - template: ./.azure-pipelines/azure-pipelines-osx.yml \ No newline at end of file diff --git a/build-locally.py b/build-locally.py index 05493e4d8..3beb68ccd 100755 --- a/build-locally.py +++ b/build-locally.py @@ -98,7 +98,10 @@ def main(args=None): p.add_argument( "--debug", action="store_true", - help="Setup debug environment using `conda debug`", + help=( + "Setup debug environment using `conda debug` " + "(or `rattler-build debug` for rattler-build recipes)" + ), ) p.add_argument("--output-id", help="If running debug, specify the output to setup.")