diff --git a/.github/workflows/generate_matrix.py b/.github/workflows/generate_matrix.py index 47e0685f0..18ea36898 100755 --- a/.github/workflows/generate_matrix.py +++ b/.github/workflows/generate_matrix.py @@ -95,6 +95,7 @@ def generate_matrix(git_ref, review, event_name=""): matrix.append( { "python_version": f"3.{PYTHON_MINOR_VERSION_MIN}", + "runner": "windows-latest", "skip": run_no_jobs, } ) @@ -102,8 +103,9 @@ def generate_matrix(git_ref, review, event_name=""): matrix.append( { "test_command": "uv run tox -e nopytz", - "test_name": f"3.{PYTHON_MINOR_VERSION_MIN} (nopytz)", + "qualifier": "nopytz", "python_version": f"3.{PYTHON_MINOR_VERSION_MIN}", + "runner": "macos-latest", "skip": run_no_jobs, } ) @@ -111,6 +113,7 @@ def generate_matrix(git_ref, review, event_name=""): matrix.append( { "python_version": f"3.{PYTHON_MINOR_VERSION_MAX}", + "runner": "ubuntu-latest", "skip": run_no_jobs, } ) @@ -125,16 +128,40 @@ def generate_matrix(git_ref, review, event_name=""): matrix.append( { "python_version": f"3.{minor_python_version}", + "runner": "ubuntu-latest", "skip": not run_all_jobs, } ) + matrix.append( + { + "python_version": f"3.{PYTHON_MINOR_VERSION_MIN}", + "runner": "ubuntu-latest", + "skip": not run_all_jobs, + } + ) + matrix.append( + { + "python_version": f"3.{PYTHON_MINOR_VERSION_MAX}", + "runner": "windows-latest", + "skip": not run_all_jobs, + } + ) + matrix.append( + { + "python_version": f"3.{PYTHON_MINOR_VERSION_MAX}", + "runner": "macos-latest", + "skip": not run_all_jobs, + } + ) + # pypy is slow but good to test matrix.insert( 0, { + "label": "pypy3", "python_version": "pypy3.11", - "test_name": "pypy3", + "runner": "ubuntu-latest", "skip": not runs_pypy, }, ) @@ -148,8 +175,14 @@ def generate_matrix(git_ref, review, event_name=""): # for run in matrix: - run.setdefault("test_name", run["python_version"]) run.setdefault("test_command", COMMAND_TEST) + label = run.pop("label", run["python_version"]) + details = filter(None, (run.pop("qualifier", ""), run["runner"])) + run["test_name"] = f"{label} ({', '.join(details)})" + if run["skip"]: + # A skipped job still starts a runner. Keep it on Ubuntu so only + # real runs spend macOS or Windows minutes. + run["runner"] = "ubuntu-latest" include = list(matrix) skipped = [run["test_name"] for run in matrix if run["skip"]] diff --git a/.github/workflows/test_generate_matrix.py b/.github/workflows/test_generate_matrix.py index 2460cc6cf..0105dc146 100644 --- a/.github/workflows/test_generate_matrix.py +++ b/.github/workflows/test_generate_matrix.py @@ -9,9 +9,26 @@ import pytest from generate_matrix import generate_matrix -CASES_ALL = {"3.10", "3.11", "3.12", "3.13", "3.14", "3.10 (nopytz)", "pypy3"} -CASES_MIN = {"3.10", "3.14", "3.10 (nopytz)"} -CASES_NO_PYPY = CASES_ALL - {"pypy3"} +RUNNERS = {"ubuntu-latest", "windows-latest", "macos-latest"} + +CASES_ALL = { + "3.10 (ubuntu-latest)", + "3.10 (windows-latest)", + "3.11 (ubuntu-latest)", + "3.12 (ubuntu-latest)", + "3.13 (ubuntu-latest)", + "3.14 (ubuntu-latest)", + "3.14 (windows-latest)", + "3.14 (macos-latest)", + "3.10 (nopytz, macos-latest)", + "pypy3 (ubuntu-latest)", +} +CASES_MIN = { + "3.10 (windows-latest)", + "3.14 (ubuntu-latest)", + "3.10 (nopytz, macos-latest)", +} +CASES_NO_PYPY = CASES_ALL - {"pypy3 (ubuntu-latest)"} CASES_0 = set() @@ -127,6 +144,7 @@ def test_all_cases_are_always_included(running_names, skipped_names, matrix): [ "test_name", "python_version", + "runner", "skip", "test_command", ], @@ -136,6 +154,37 @@ def test_parameters_are_present(matrix, attribute): assert attribute in case, f"Missing {attribute} in {case}" +def test_runners_are_supported(matrix): + """Only runners that exist are used, so no job waits for a typo.""" + for case in matrix["include"]: + assert case["runner"] in RUNNERS, case + + +def test_skipped_jobs_run_on_ubuntu(matrix): + """Skipped jobs still start a runner: do not spend macOS or Windows minutes.""" + for case in matrix["include"]: + if case["skip"]: + assert case["runner"] == "ubuntu-latest", case + + +def test_minimal_set_covers_all_operating_systems(): + """A pull request tests on Linux, Windows and macOS.""" + matrix = generate_matrix("refs/heads/pr-branch", "", "pull_request") + running = {case["runner"] for case in matrix["include"] if not case["skip"]} + assert running == RUNNERS + + +def test_full_run_covers_every_version_on_ubuntu(): + """Every supported Python version runs on Ubuntu when all jobs run.""" + matrix = generate_matrix("refs/heads/main", "", "push") + versions = { + case["python_version"] + for case in matrix["include"] + if not case["skip"] and case["runner"] == "ubuntu-latest" + } + assert {"3.10", "3.11", "3.12", "3.13", "3.14"} <= versions + + @pytest.mark.parametrize( ("event_name", "git_ref", "expected_skip"), [ @@ -152,5 +201,9 @@ def test_parameters_are_present(matrix, attribute): def test_pypy_only_runs_on_push_to_main(event_name, git_ref, expected_skip): """Run the slow PyPy job after changes are merged or pushed to main.""" matrix = generate_matrix(git_ref, "", event_name) - pypy = next(case for case in matrix["include"] if case["test_name"] == "pypy3") + pypy = next( + case + for case in matrix["include"] + if case["test_name"] == "pypy3 (ubuntu-latest)" + ) assert pypy["skip"] is expected_skip diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3cc3746a..94dec8c25 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,6 +22,16 @@ jobs: include: ${{ steps.set-matrix.outputs.include }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-tags: true + - name: Install uv and Python 3.14 + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + version: "${{ vars.UV_VERSION }}" + python-version: "3.14" + - name: Test the matrix generator + run: uv run --no-dev --group test pytest .github/workflows/test_generate_matrix.py - name: Set matrix id: set-matrix env: @@ -38,7 +48,7 @@ jobs: matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.include) }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} name: ${{ matrix.test_name }} steps: - name: Skip notice @@ -63,13 +73,14 @@ jobs: - name: Install dependencies if: ${{ !matrix.skip }} run: | - make dev PYTHONVERSION=${{ matrix.python_version }} + uv venv --python ${{ matrix.python_version }} + uv sync --group dev uv tree - name: Test if: ${{ !matrix.skip }} run: ${{ matrix.test_command }} - name: Coveralls Parallel - if: ${{ !matrix.skip }} + if: ${{ !matrix.skip && matrix.runner == 'ubuntu-latest' }} uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2 with: flag-name: run-${{ matrix.test_name }} diff --git a/docs/contribute/maintenance.rst b/docs/contribute/maintenance.rst index d2f03d607..83a24c325 100644 --- a/docs/contribute/maintenance.rst +++ b/docs/contribute/maintenance.rst @@ -388,9 +388,10 @@ Updating Python versions When adding support for a new Python version, or removing support for an old one, the following files need to be updated: :file:`.github/workflows/tests.yml` - Upgrade Python versions. -:file:`.github/workflows/*.py` - Add or remove the Python version from the test matrix. + Keep the test jobs in sync with the matrix, for example the runners. +:file:`.github/workflows/generate_matrix.py` + Add or remove the Python version from the test matrix and choose the + runner each job uses. :file:`tox.ini` Update the ``envlist`` to include or remove the Python version. :file:`pyproject.toml` diff --git a/news/1062.internal b/news/1062.internal new file mode 100644 index 000000000..300fada71 --- /dev/null +++ b/news/1062.internal @@ -0,0 +1 @@ +Ran the test matrix on Windows and macOS as well as Ubuntu, with one job per operating system on pull requests and every supported Python version on Ubuntu for full runs. Reviewed by GPT-6 Luna. @SemTiOne