Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions .github/workflows/generate_matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,22 +95,25 @@ def generate_matrix(git_ref, review, event_name=""):
matrix.append(
{
"python_version": f"3.{PYTHON_MINOR_VERSION_MIN}",
"runner": "windows-latest",
"skip": run_no_jobs,
}
)
# RULE: use lowest allowed version for nopytz
matrix.append(
{
"test_command": "uv run tox -e nopytz",
"test_name": f"3.{PYTHON_MINOR_VERSION_MIN} (nopytz)",
"qualifier": "nopytz",
"python_version": f"3.{PYTHON_MINOR_VERSION_MIN}",
"runner": "macos-latest",
"skip": run_no_jobs,
}
)
# RULE: always test the latest version
matrix.append(
{
"python_version": f"3.{PYTHON_MINOR_VERSION_MAX}",
"runner": "ubuntu-latest",
"skip": run_no_jobs,
}
)
Expand All @@ -125,16 +128,40 @@ def generate_matrix(git_ref, review, event_name=""):
matrix.append(
{
"python_version": f"3.{minor_python_version}",
"runner": "ubuntu-latest",
"skip": not run_all_jobs,
}
)

matrix.append(
{
"python_version": f"3.{PYTHON_MINOR_VERSION_MIN}",
"runner": "ubuntu-latest",
"skip": not run_all_jobs,
}
)
matrix.append(
{
"python_version": f"3.{PYTHON_MINOR_VERSION_MAX}",
"runner": "windows-latest",
"skip": not run_all_jobs,
}
)
matrix.append(
{
"python_version": f"3.{PYTHON_MINOR_VERSION_MAX}",
"runner": "macos-latest",
"skip": not run_all_jobs,
}
)

# pypy is slow but good to test
matrix.insert(
0,
{
"label": "pypy3",
"python_version": "pypy3.11",
"test_name": "pypy3",
"runner": "ubuntu-latest",
"skip": not runs_pypy,
},
)
Expand All @@ -148,8 +175,14 @@ def generate_matrix(git_ref, review, event_name=""):
#

for run in matrix:
run.setdefault("test_name", run["python_version"])
run.setdefault("test_command", COMMAND_TEST)
label = run.pop("label", run["python_version"])
details = filter(None, (run.pop("qualifier", ""), run["runner"]))
run["test_name"] = f"{label} ({', '.join(details)})"
if run["skip"]:
# A skipped job still starts a runner. Keep it on Ubuntu so only
# real runs spend macOS or Windows minutes.
run["runner"] = "ubuntu-latest"

include = list(matrix)
skipped = [run["test_name"] for run in matrix if run["skip"]]
Expand Down
61 changes: 57 additions & 4 deletions .github/workflows/test_generate_matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,26 @@
import pytest
from generate_matrix import generate_matrix

CASES_ALL = {"3.10", "3.11", "3.12", "3.13", "3.14", "3.10 (nopytz)", "pypy3"}
CASES_MIN = {"3.10", "3.14", "3.10 (nopytz)"}
CASES_NO_PYPY = CASES_ALL - {"pypy3"}
RUNNERS = {"ubuntu-latest", "windows-latest", "macos-latest"}

CASES_ALL = {
"3.10 (ubuntu-latest)",
"3.10 (windows-latest)",
"3.11 (ubuntu-latest)",
"3.12 (ubuntu-latest)",
"3.13 (ubuntu-latest)",
"3.14 (ubuntu-latest)",
"3.14 (windows-latest)",
"3.14 (macos-latest)",
"3.10 (nopytz, macos-latest)",
"pypy3 (ubuntu-latest)",
}
CASES_MIN = {
"3.10 (windows-latest)",
"3.14 (ubuntu-latest)",
"3.10 (nopytz, macos-latest)",
}
CASES_NO_PYPY = CASES_ALL - {"pypy3 (ubuntu-latest)"}
CASES_0 = set()


Expand Down Expand Up @@ -127,6 +144,7 @@ def test_all_cases_are_always_included(running_names, skipped_names, matrix):
[
"test_name",
"python_version",
"runner",
"skip",
"test_command",
],
Expand All @@ -136,6 +154,37 @@ def test_parameters_are_present(matrix, attribute):
assert attribute in case, f"Missing {attribute} in {case}"


def test_runners_are_supported(matrix):
"""Only runners that exist are used, so no job waits for a typo."""
for case in matrix["include"]:
assert case["runner"] in RUNNERS, case


def test_skipped_jobs_run_on_ubuntu(matrix):
"""Skipped jobs still start a runner: do not spend macOS or Windows minutes."""
for case in matrix["include"]:
if case["skip"]:
assert case["runner"] == "ubuntu-latest", case


def test_minimal_set_covers_all_operating_systems():
"""A pull request tests on Linux, Windows and macOS."""
matrix = generate_matrix("refs/heads/pr-branch", "", "pull_request")
running = {case["runner"] for case in matrix["include"] if not case["skip"]}
assert running == RUNNERS


def test_full_run_covers_every_version_on_ubuntu():
"""Every supported Python version runs on Ubuntu when all jobs run."""
matrix = generate_matrix("refs/heads/main", "", "push")
versions = {
case["python_version"]
for case in matrix["include"]
if not case["skip"] and case["runner"] == "ubuntu-latest"
}
assert {"3.10", "3.11", "3.12", "3.13", "3.14"} <= versions


@pytest.mark.parametrize(
("event_name", "git_ref", "expected_skip"),
[
Expand All @@ -152,5 +201,9 @@ def test_parameters_are_present(matrix, attribute):
def test_pypy_only_runs_on_push_to_main(event_name, git_ref, expected_skip):
"""Run the slow PyPy job after changes are merged or pushed to main."""
matrix = generate_matrix(git_ref, "", event_name)
pypy = next(case for case in matrix["include"] if case["test_name"] == "pypy3")
pypy = next(
case
for case in matrix["include"]
if case["test_name"] == "pypy3 (ubuntu-latest)"
)
assert pypy["skip"] is expected_skip
17 changes: 14 additions & 3 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,16 @@ jobs:
include: ${{ steps.set-matrix.outputs.include }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-tags: true
- name: Install uv and Python 3.14
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
enable-cache: false
version: "${{ vars.UV_VERSION }}"
python-version: "3.14"
- name: Test the matrix generator
run: uv run --no-dev --group test pytest .github/workflows/test_generate_matrix.py
- name: Set matrix
id: set-matrix
env:
Expand All @@ -38,7 +48,7 @@ jobs:
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.include) }}

runs-on: ubuntu-latest
runs-on: ${{ matrix.runner }}
name: ${{ matrix.test_name }}
steps:
- name: Skip notice
Expand All @@ -63,13 +73,14 @@ jobs:
- name: Install dependencies
if: ${{ !matrix.skip }}
run: |
make dev PYTHONVERSION=${{ matrix.python_version }}
uv venv --python ${{ matrix.python_version }}
uv sync --group dev
uv tree
- name: Test
if: ${{ !matrix.skip }}
run: ${{ matrix.test_command }}
- name: Coveralls Parallel
if: ${{ !matrix.skip }}
if: ${{ !matrix.skip && matrix.runner == 'ubuntu-latest' }}
uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2
with:
flag-name: run-${{ matrix.test_name }}
Expand Down
7 changes: 4 additions & 3 deletions docs/contribute/maintenance.rst
Original file line number Diff line number Diff line change
Expand Up @@ -388,9 +388,10 @@ Updating Python versions
When adding support for a new Python version, or removing support for an old one, the following files need to be updated:

:file:`.github/workflows/tests.yml`
Upgrade Python versions.
:file:`.github/workflows/*.py`
Add or remove the Python version from the test matrix.
Keep the test jobs in sync with the matrix, for example the runners.
:file:`.github/workflows/generate_matrix.py`
Add or remove the Python version from the test matrix and choose the
runner each job uses.
:file:`tox.ini`
Update the ``envlist`` to include or remove the Python version.
:file:`pyproject.toml`
Expand Down
1 change: 1 addition & 0 deletions news/1062.internal
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Ran the test matrix on Windows and macOS as well as Ubuntu, with one job per operating system on pull requests and every supported Python version on Ubuntu for full runs. Reviewed by GPT-6 Luna. @SemTiOne
Loading