execute_matrix.yml

name: Execute matrix

permissions:
  contents: read

on:
  workflow_call:
    inputs:
      name:
        type: string
        description: "Name used for the concurrency group. Set this when a workflow calls execute_matrix.yml more than once, or the calls cancel each other."
        required: true
      matrix_yaml_string:
        type: string
        description: "The YAML string containing the matrix definition."
        required: true
      command:
        type: string
        description: "Command to run for matrix entries that do not specify their own. Lets a caller pair a toolchain matrix with its own command."
        default: ""
      setup_command:
        type: string
        description: "Setup command for matrix entries that do not specify their own."
        default: ""
      command_arguments:
        type: string
        description: "Space-separated arguments appended to entries that do not specify their own."
        default: ""
      env:
        type: string
        description: "Environment variables (JSON or YAML object) for every entry. Per-entry values win on a key-by-key basis."
        default: "{}"
      job_timeout:
        type: number
        description: "Timeout in minutes for each job, for entries that do not carry their own timeout_minutes."
        default: 90
      needs_token:
        type: boolean
        description: "Whether to provide GITHUB_TOKEN to jobs"
        default: false
      enable_cross_pr_testing:
        type: boolean
        description: "Whether PRs can be tested together with linked PRs mentioned in the PR description"
        default: false

      workflows_repository:
        type: string
        description: "Repository to take the matrix scripts from. Point this at a fork to test a change to the workflows before it lands; it carries no version, so Dependabot has only the `uses:` line to bump."
        default: "swiftlang/github-workflows"
      workflows_ref:
        type: string
        description: "Ref to take the scripts from. Empty uses workflows_repository's default branch, which is correct for a released version; set it when workflows_repository is a fork whose default branch does not carry the change."
        default: ""
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.name }}
  cancel-in-progress: true

jobs:
  convert-matrix:
    name: Convert matrix YAML to JSON
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.convert.outputs.matrix }}
      job_count: ${{ steps.convert.outputs.job_count }}
    steps:
      - name: Convert YAML to JSON
        id: convert
        env:
          MATRIX_YAML: ${{ inputs.matrix_yaml_string }}
          DEFAULT_COMMAND: ${{ inputs.command }}
          DEFAULT_SETUP_COMMAND: ${{ inputs.setup_command }}
          DEFAULT_COMMAND_ARGUMENTS: ${{ inputs.command_arguments }}
          DEFAULT_ENV: ${{ inputs.env }}
        run: |
          set -euo pipefail

          if [ -z "${MATRIX_YAML//[[:space:]]/}" ]; then
            echo "::error::matrix_yaml_string is empty. Pass the matrix YAML; a caller that wants no jobs passes 'config: []'."
            exit 1
          fi

          if ! parsed=$(echo "$MATRIX_YAML" | yq -o=json '.'); then
            echo "::error::matrix_yaml_string is not valid YAML."
            exit 1
          fi

          # A matrix with no entries is valid but a missing or misspelled 'config' key is not.
          if ! echo "$parsed" | jq -e 'type == "object" and has("config")' > /dev/null; then
            echo "::error::matrix_yaml_string has no 'config' key. It must be a map with a 'config' list; pass 'config: []' for no jobs."
            exit 1
          fi
          if ! echo "$parsed" | jq -e '.config | type == "array"' > /dev/null; then
            echo "::error::'config' must be a list of matrix entries."
            exit 1
          fi

          default_env=$(echo "$DEFAULT_ENV" | yq -o=json '. // {}')

          # Fill in what an entry leaves out, so a toolchain-only matrix can be paired
          # with a command by its caller.
          matrix=$(echo "$parsed" | jq -c \
            --arg command "$DEFAULT_COMMAND" \
            --arg setup_command "$DEFAULT_SETUP_COMMAND" \
            --arg command_arguments "$DEFAULT_COMMAND_ARGUMENTS" \
            --argjson default_env "$default_env" \
            '{config: [
                .config[]
                | if (.command // "") == "" then .command = $command else . end
                | if (.setup_command // "") == "" then .setup_command = $setup_command else . end
                | if ((.command_arguments // []) | length) == 0
                  then .command_arguments = ($command_arguments | split(" ") | map(select(. != "")))
                  else . end
                | .env = ($default_env + (.env // {}))
             ]}')

          missing=$(echo "$matrix" | jq -r '[.config[] | select((.command // "") == "") | .name] | join(", ")')
          if [ -n "$missing" ]; then
            echo "::error::No command for matrix entries: $missing. Set it on the entry or pass the workflow's command input."
            exit 1
          fi

          # Reject unexpected host platforms early.
          unsupported=$(echo "$matrix" | jq -r '
            [.config[]
             | select((.platform // "") as $p
                      | ["Linux", "macOS", "Windows", "FreeBSD"] | index($p) == null)
             | "\(.name // "<unnamed>") (platform: \(.platform // "<none>"))"]
            | join(", ")')
          if [ -n "$unsupported" ]; then
            echo "::error::Unsupported platform for matrix entries: $unsupported. Must be one of Linux, macOS, Windows, FreeBSD."
            exit 1
          fi

          job_count=$(echo "$matrix" | jq -r '.config | length')
          if [ "$job_count" -eq 0 ]; then
            echo "::notice::No matrix entries; no jobs will run."
          else
            echo "::notice::Running $job_count job(s): $(echo "$matrix" | jq -r '[.config[].name] | join(", ")')"
          fi
          echo "job_count=$job_count" >> $GITHUB_OUTPUT

          echo "matrix<<MATRIX_EOF" >> $GITHUB_OUTPUT
          echo "$matrix" >> $GITHUB_OUTPUT
          echo "MATRIX_EOF" >> $GITHUB_OUTPUT

  execute-matrix:
    name: ${{ matrix.config.name }}
    needs: convert-matrix
    if: ${{ needs.convert-matrix.outputs.job_count != '0' }}
    runs-on: ${{ matrix.config.runner }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.convert-matrix.outputs.matrix) }}
    env:
      SWIFT_VERSION: >-
        ${{ matrix.config.swift_build.swift_version
            || matrix.config.xcode_build.swift_version
            || matrix.config.xcode_build.xcode_version
            || matrix.config.freebsd.swift_version
            || '' }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
        with:
          persist-credentials: false
          submodules: true

      - name: Resolve the workflows source
        id: workflows_source
        shell: bash
        env:
          WORKFLOWS_REPOSITORY: ${{ inputs.workflows_repository }}
          WORKFLOWS_REF: ${{ inputs.workflows_ref }}
        run: |
          set -euo pipefail

          # Empty uses the default branch.
          echo "ref=$WORKFLOWS_REF" >> $GITHUB_OUTPUT

          if [ "$GITHUB_REPOSITORY" = "$WORKFLOWS_REPOSITORY" ]; then
            echo "needs_checkout=false" >> $GITHUB_OUTPUT
            root="$GITHUB_WORKSPACE"
            scripts_relative=".github/workflows/scripts"
          else
            echo "needs_checkout=true" >> $GITHUB_OUTPUT
            root="$GITHUB_WORKSPACE/github-workflows"
            scripts_relative="github-workflows/.github/workflows/scripts"
          fi

          echo "root_directory=$root" >> $GITHUB_OUTPUT
          # The Windows dispatch invokes the runner through PowerShell, which
          # needs backslashes.
          echo "root_directory_windows=$(echo "$root" | sed 's|/|\\|g')" >> $GITHUB_OUTPUT
          # The FreeBSD VM works in its own copy of the workspace, at a path the host
          # does not know, so it rebuilds ${SCRIPTS_ROOT} from this relative form.
          echo "scripts_root_relative=$scripts_relative" >> $GITHUB_OUTPUT

      - name: Checkout the workflows repository
        if: ${{ steps.workflows_source.outputs.needs_checkout == 'true' }}
        uses: actions/checkout@v7
        with:
          repository: ${{ inputs.workflows_repository }}
          ref: ${{ steps.workflows_source.outputs.ref }}
          path: github-workflows
          persist-credentials: false

      # Linux and Windows install their toolchain inside the job runner, so they
      # check out linked PRs there afterwards, to compile the checkout script with
      # the toolchain under test (and inside the container when there is one).
      # macOS uses the host's Xcode, which is there before the job starts, so
      # there is nothing to wait for and the checkout is done here.
      - name: Check out linked PRs (macOS)
        if: ${{ inputs.enable_cross_pr_testing && github.event_name == 'pull_request' && matrix.config.platform == 'macOS' }}
        shell: bash
        env:
          SCRIPTS_ROOT: ${{ steps.workflows_source.outputs.root_directory }}/.github/workflows/scripts
          PR_REPO: ${{ github.repository }}
          PR_NUMBER: ${{ github.event.number }}
        run: |
          cp "$SCRIPTS_ROOT/cross-pr-checkout.swift" /tmp/cross-pr-checkout.swift
          swift /tmp/cross-pr-checkout.swift "$PR_REPO" "$PR_NUMBER"

      - name: Run matrix job (Linux)
        if: ${{ matrix.config.platform == 'Linux' }}
        timeout-minutes: ${{ matrix.config.timeout_minutes || inputs.job_timeout }}
        shell: bash
        env:
          GITHUB_TOKEN: ${{ inputs.needs_token && secrets.GITHUB_TOKEN || '' }}
          SCRIPTS_ROOT: ${{ steps.workflows_source.outputs.root_directory }}/.github/workflows/scripts
          CONTAINER_JSON: ${{ toJson(matrix.config.swift_build.container) }}
          MATRIX_SWIFT_VERSION: ${{ matrix.config.swift_build.swift_version }}
          MATRIX_TOOLCHAIN: ${{ matrix.config.swift_build.toolchain }}
          MATRIX_SWIFTLY: ${{ matrix.config.swift_build.swiftly }}
          MATRIX_SETUP_COMMAND: ${{ matrix.config.setup_command }}
          MATRIX_COMMAND: ${{ matrix.config.command }}
          MATRIX_COMMAND_ARGUMENTS: ${{ toJson(matrix.config.command_arguments) }}
          MATRIX_ENV: ${{ toJson(matrix.config.env) }}
          NEEDS_TOKEN: ${{ inputs.needs_token }}
          MATRIX_SDK: ${{ toJson(matrix.config.swift_build.sdk) }}
          CROSS_PR_TESTING: ${{ inputs.enable_cross_pr_testing && github.event_name == 'pull_request' }}
          CROSS_PR_REPO: ${{ github.repository }}
          CROSS_PR_NUMBER: ${{ github.event.number }}
        run: |
          "$SCRIPTS_ROOT/matrix/job-runner-linux.sh" \
            "$MATRIX_SWIFT_VERSION" \
            "$MATRIX_SETUP_COMMAND" \
            "$MATRIX_COMMAND" \
            "$MATRIX_COMMAND_ARGUMENTS" \
            "$MATRIX_ENV" \
            "$NEEDS_TOKEN" \
            "$MATRIX_SDK"

      - name: Enable KVM and free disk space
        if: ${{ matrix.config.android_emulator == true && matrix.config.platform == 'Linux' }}
        shell: bash
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm
          sudo rm -rf /opt/microsoft /opt/google /opt/az /opt/ghc /usr/share/dotnet /usr/local/share/boost /opt/hostedtoolcache /usr/local/share/chromium
          df -h

      - name: Run Android emulator tests
        if: ${{ matrix.config.android_emulator == true && matrix.config.platform == 'Linux' }}
        timeout-minutes: ${{ matrix.config.timeout_minutes || inputs.job_timeout }}
        shell: bash
        env:
          SCRIPTS_ROOT: ${{ steps.workflows_source.outputs.root_directory }}/.github/workflows/scripts
          SDK_TRIPLES: ${{ join(matrix.config.swift_build.sdk.triples, ' --android-sdk-triple=') }}
        run: |
          # The SDK build exported the matched toolchain and SDK bundle to GITHUB_ENV for
          # this script.
          # Intentional word splitting to pass multiple --android-sdk-triple arguments
          # shellcheck disable=SC2086
          "$SCRIPTS_ROOT/android/android-emulator-tests.sh" \
            --android-sdk-triple=$SDK_TRIPLES

      - name: Run matrix job (Windows)
        if: ${{ matrix.config.platform == 'Windows' }}
        timeout-minutes: ${{ matrix.config.timeout_minutes || inputs.job_timeout }}
        env:
          GITHUB_TOKEN: ${{ inputs.needs_token && secrets.GITHUB_TOKEN || '' }}
          SCRIPTS_ROOT: ${{ steps.workflows_source.outputs.root_directory_windows }}\.github\workflows\scripts
          CONTAINER_IMAGE: ${{ matrix.config.swift_build.container.image }}
          CROSS_PR_TESTING: ${{ inputs.enable_cross_pr_testing && github.event_name == 'pull_request' }}
          CROSS_PR_REPO: ${{ github.repository }}
          CROSS_PR_NUMBER: ${{ github.event.number }}
          MATRIX_SWIFT_VERSION: ${{ matrix.config.swift_build.swift_version }}
          MATRIX_TOOLCHAIN: ${{ matrix.config.swift_build.toolchain }}
          MATRIX_SETUP_COMMAND: ${{ matrix.config.setup_command }}
          MATRIX_COMMAND: ${{ matrix.config.command }}
          MATRIX_COMMAND_ARGUMENTS: ${{ toJson(matrix.config.command_arguments) }}
          MATRIX_ENV: ${{ toJson(matrix.config.env) }}
          NEEDS_TOKEN: ${{ inputs.needs_token }}
        shell: pwsh
        run: |
          & "$env:SCRIPTS_ROOT\matrix\job-runner-windows.ps1" `
            -SwiftVersion "$env:MATRIX_SWIFT_VERSION" `
            -SetupCommand "$env:MATRIX_SETUP_COMMAND" `
            -Command "$env:MATRIX_COMMAND" `
            -CommandArguments $env:MATRIX_COMMAND_ARGUMENTS `
            -EnvJson $env:MATRIX_ENV `
            -NeedsToken "$env:NEEDS_TOKEN"

      - name: Run matrix job (macOS)
        if: ${{ matrix.config.platform == 'macOS' }}
        timeout-minutes: ${{ matrix.config.timeout_minutes || inputs.job_timeout }}
        shell: bash
        env:
          GITHUB_TOKEN: ${{ inputs.needs_token && secrets.GITHUB_TOKEN || '' }}
          SCRIPTS_ROOT: ${{ steps.workflows_source.outputs.root_directory }}/.github/workflows/scripts
          XCODE_TARGETS_JSON: ${{ toJson(matrix.config.xcode_build.targets) }}
          XCODE_DEBUG_OUTPUT: ${{ matrix.config.xcode_build.debug_output }}
          SWIFTLY_TOOLCHAIN: ${{ matrix.config.xcode_build.swiftly_toolchain }}
          MATRIX_XCODE_VERSION: ${{ matrix.config.xcode_build.xcode_version }}
          MATRIX_SWIFT_VERSION: ${{ matrix.config.xcode_build.swift_version }}
          MATRIX_SETUP_COMMAND: ${{ matrix.config.setup_command }}
          MATRIX_COMMAND: ${{ matrix.config.command }}
          MATRIX_COMMAND_ARGUMENTS: ${{ toJson(matrix.config.command_arguments) }}
          MATRIX_ENV: ${{ toJson(matrix.config.env) }}
          NEEDS_TOKEN: ${{ inputs.needs_token }}
        run: |
          "$SCRIPTS_ROOT/matrix/job-runner-macos.sh" \
            "$MATRIX_XCODE_VERSION" \
            "$MATRIX_SWIFT_VERSION" \
            "$MATRIX_SETUP_COMMAND" \
            "$MATRIX_COMMAND" \
            "$MATRIX_COMMAND_ARGUMENTS" \
            "$MATRIX_ENV" \
            "$NEEDS_TOKEN"

      - name: Merge the FreeBSD environment
        id: freebsd_env
        if: ${{ matrix.config.platform == 'FreeBSD' }}
        shell: bash
        env:
          FREEBSD_ENV_VARS: ${{ matrix.config.freebsd.env_vars }}
          MATRIX_ENV: ${{ toJson(matrix.config.env) }}
        run: |
          set -euo pipefail

          # The VM reads its environment as KEY=VALUE lines, so a value that spans lines
          # would arrive cut short.
          if echo "$MATRIX_ENV" | jq -e '(. // {}) | any(.[]; tostring | test("\n"))' > /dev/null; then
            echo "::error::A FreeBSD entry's env holds a value spanning several lines, which the VM cannot receive."
            exit 1
          fi

          # The VM has no jq, so the entry's env is flattened here into the KEY=VALUE
          # lines the VM already reads. It is applied last, so an entry's own value wins
          # over freebsd.env_vars.
          merged=$( {
            printf '%s\n' "$FREEBSD_ENV_VARS"
            echo "$MATRIX_ENV" | jq -r '(. // {}) | to_entries[] | "\(.key)=\(.value)"'
          } | awk 'NF')

          {
            echo "env_vars<<FREEBSD_ENV_EOF"
            if [ -n "$merged" ]; then
              printf '%s\n' "$merged"
            fi
            echo "FREEBSD_ENV_EOF"
          } >> $GITHUB_OUTPUT

      - name: Quote the FreeBSD command arguments
        id: freebsd_arguments
        if: ${{ matrix.config.platform == 'FreeBSD' }}
        shell: bash
        env:
          MATRIX_COMMAND_ARGUMENTS: ${{ toJson(matrix.config.command_arguments) }}
        run: |
          set -euo pipefail

          # The VM has no jq and eval's the command it runs, so each argument is quoted
          # here: an argument containing whitespace would otherwise arrive as several.
          # A string, which a hand-written entry may give instead of a list, is already in
          # the form the VM receives.
          quoted=$(echo "$MATRIX_COMMAND_ARGUMENTS" \
            | jq -r '(. // []) | if type == "array" then map(@sh) | join(" ") else tostring end')

          # An argument may contain a newline, which a KEY=VALUE output line cannot carry.
          {
            echo "command_arguments<<FREEBSD_ARGUMENTS_EOF"
            if [ -n "$quoted" ]; then
              printf '%s\n' "$quoted"
            fi
            echo "FREEBSD_ARGUMENTS_EOF"
          } >> $GITHUB_OUTPUT

      - name: Run matrix job (FreeBSD)
        if: ${{ matrix.config.platform == 'FreeBSD' }}
        timeout-minutes: ${{ matrix.config.timeout_minutes || inputs.job_timeout }}
        uses: vmactions/freebsd-vm@v1
        env:
          SWIFT_WEB_URL: ${{ matrix.config.freebsd.swift_url }}
          BUILD_FLAGS: ${{ matrix.config.freebsd.build_flags }}
          FREEBSD_ENV_VARS: ${{ steps.freebsd_env.outputs.env_vars }}
          MATRIX_SETUP_COMMAND: ${{ matrix.config.setup_command }}
          MATRIX_COMMAND: ${{ matrix.config.command }}
          # Shell-quoted on the host, where jq is available.
          MATRIX_COMMAND_ARGUMENTS: ${{ steps.freebsd_arguments.outputs.command_arguments }}
          SCRIPTS_ROOT_RELATIVE: ${{ steps.workflows_source.outputs.scripts_root_relative }}
          GITHUB_TOKEN: ${{ inputs.needs_token && secrets.GITHUB_TOKEN || '' }}
          CROSS_PR_TESTING: ${{ inputs.enable_cross_pr_testing && github.event_name == 'pull_request' }}
          CROSS_PR_REPO: ${{ github.repository }}
          CROSS_PR_NUMBER: ${{ github.event.number }}
        with:
          envs: 'SWIFT_WEB_URL BUILD_FLAGS GITHUB_TOKEN FREEBSD_ENV_VARS MATRIX_SETUP_COMMAND MATRIX_COMMAND MATRIX_COMMAND_ARGUMENTS SCRIPTS_ROOT_RELATIVE CROSS_PR_TESTING CROSS_PR_REPO CROSS_PR_NUMBER'
          release: "${{ matrix.config.freebsd.os_version }}"
          arch: "x86_64"
          sync: rsync
          copyback: false
          usesh: true
          prepare: |
            fetch -o /tmp/swift.tar.gz "$SWIFT_WEB_URL"
            mkdir -p /opt/swift
            tar -xzf /tmp/swift.tar.gz -C /opt/swift
            pkg install -y git sqlite3 libuuid python3 curl brotli bash
            git config --global init.defaultBranch 'main'
            /opt/swift/usr/bin/swift --version
          run: |
            # The VM runs this under sh with no -e, so a failing setup command or
            # cross-PR checkout would otherwise be followed by the build anyway,
            # testing the wrong tree and reporting success.
            set -e
            export PATH="/opt/swift/usr/bin:$PATH"
            swift --version
            # The command runs in the VM's copy of the workspace, so ${SCRIPTS_ROOT} is
            # this copy's path, not the host's.
            SCRIPTS_ROOT="$(pwd)/$SCRIPTS_ROOT_RELATIVE"
            export SCRIPTS_ROOT
            if [ -n "$FREEBSD_ENV_VARS" ]; then
              printf '%s\n' "$FREEBSD_ENV_VARS" > /tmp/.env_vars
              while IFS= read -r _line || [ -n "$_line" ]; do
                if [ -n "$_line" ]; then
                  export "$_line"
                fi
              done < /tmp/.env_vars
            fi
            if [ "$CROSS_PR_TESTING" = "true" ]; then
              cat "$SCRIPTS_ROOT/cross-pr-checkout.swift" > /tmp/cross-pr-checkout.swift
              swift /tmp/cross-pr-checkout.swift "$CROSS_PR_REPO" "$CROSS_PR_NUMBER"
            fi
            if [ -n "$MATRIX_SETUP_COMMAND" ]; then
              eval "$MATRIX_SETUP_COMMAND"
            fi
            eval "$MATRIX_COMMAND $BUILD_FLAGS $MATRIX_COMMAND_ARGUMENTS"