beam/.github/workflows/beam_PreCommit_Python_ML.yml at master · apache/beam · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: PreCommit Python ML tests with ML deps installed
on:
  pull_request:
    branches: [ "master", "release-*" ]
    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python_ML.json']
  push:
    tags: ['v*']
    branches: ['master', 'release-*']
    paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_Python_ML.yml"]
  schedule:
    - cron: '45 2/6 * * *'
  workflow_dispatch:

# Setting explicit permissions for the action
permissions:
  actions: write
  pull-requests: write
  checks: write
  contents: read
  deployments: read
  id-token: none
  issues: write
  discussions: read
  packages: read
  pages: read
  repository-projects: read
  security-events: read
  statuses: read

# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
  group: '${{ github.workflow }} @ ${{ github.event.pull_request.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.sender.login }}'
  cancel-in-progress: true

env:
  DEVELOCITY_ACCESS_KEY: ${{ secrets.BEAM_DEVELOCITY_ACCESS_KEY }}
  # This database resource is sandboxed for testing only
  ALLOYDB_PASSWORD: '2z,b}g_],1-,4A,X'

jobs:
  beam_PreCommit_Python_ML:
    name: ${{ matrix.job_name }} (${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
        job_name: ['beam_PreCommit_Python_ML']
        python_version: ['3.11','3.12','3.13']
        # Run on both self-hosted and GitHub-hosted runners.
        # Some tests (marked require_docker_in_docker) can't run on Beam's
        # self-hosted runners due to Docker-in-Docker environment constraint.
        # These tests will only execute on ubuntu-latest (GitHub-hosted).
        # Context: https://github.com/apache/beam/pull/35585.
        os: [[self-hosted, ubuntu-24.04, main], [ubuntu-latest]]
        exclude:
          # exercise DinD tests (needed for Milvus) for latest Python version
          - python_version: '3.11'
            os: [ubuntu-latest]
          - python_version: '3.12'
            os: [ ubuntu-latest ]
    if: |
      github.event_name == 'push' ||
      github.event_name == 'pull_request' ||
      (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
      github.event_name == 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
      - name: Free Disk Space (Ubuntu)
        if: contains(matrix.os, 'ubuntu-latest')
        uses: jlumbroso/free-disk-space@v2.0.0
        with:
          # Remove unnecessary packages to free up space
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - name: Setup repository
        uses: ./.github/actions/setup-action
      - name: Setup environment
        uses: ./.github/actions/setup-environment-action
        with:
          java-version: default
          python-version: ${{ matrix.python_version }}
      - name: Set PY_VER_CLEAN
        id: set_py_ver_clean
        run: |
          PY_VER=${{ matrix.python_version }}
          PY_VER_CLEAN=${PY_VER//.}
          echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT
      - name: Run pythonPreCommit
        uses: ./.github/actions/gradle-command-self-hosted-action
        with:
          gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:testPy${{steps.set_py_ver_clean.outputs.py_ver_clean}}ML
          arguments: |
            -Pposargs="${{
              contains(matrix.os, 'self-hosted') &&
              'apache_beam/ml/ -m (not require_docker_in_docker)' ||
                'apache_beam/ml/ -m require_docker_in_docker'
            }}" \
            -PpythonVersion=${{ matrix.python_version }}
      - name: Archive Python Test Results
        uses: actions/upload-artifact@v7
        if: failure()
        with:
          name: Python ${{ matrix.python_version }} Test Results ${{ matrix.os }}
          path: '**/pytest*.xml'
      - name: Publish Python Test Results
        uses: EnricoMi/publish-unit-test-result-action@v2
        if: always()
        with:
          commit: '${{ env.GITHUB_SHA }}'
          comment_mode: 'off'
          check_run: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
          files: '**/pytest*.xml'
          large_files: true
          check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"