Move MLX, Build Presets and macOS wheel CI to macOS 15 runners

.ci/scripts/tests/test_wheel_platform_tag.py

@@ -4,28 +4,33 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-"""Unit tests for the wheel platform tag comparison.
+"""Unit tests for the wheel platform tag checks.
 
-Here rather than in the wheel checks, because the decision under test is a pure
-function of two strings. Running it as part of the wheel checks meant it needed eleven
-built wheels to exercise one comparison, and it still could not run on a machine that
-had not built one.
+Here rather than in the wheel checks, because nothing under test needs a built wheel.
+Running it as part of the wheel checks meant it needed eleven built wheels to exercise
+one comparison, and it still could not run on a machine that had not built one.
 
-The comparison had a real defect that this covers. The release pipeline builds in a
+The tag comparison had a real defect that this covers. The release pipeline builds in a
 manylinux image and rewrites the wheel's file name, so the tag on the file and the tag
 auditwheel reports never agree in spelling, and comparing them as text rejected every
 correct wheel. No local build reproduces that rewrite, so nothing short of a unit test
 catches it before CI.
 """
 
+import subprocess
 import sys
+import zipfile
 from pathlib import Path
 
 import pytest
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "wheel"))
 
+import test_shared_libraries  # noqa: E402
 from test_shared_libraries import (  # noqa: E402
+    _mach_o_minimum,
+    _macos_minimum_violations,
+    _metallib_minimum,
     _tag_architectures_match,
     _wheel_architecture,
 )
@@ -75,3 +80,136 @@ def test_reads_every_architecture_the_project_builds_for():
     for architecture in ("x86_64", "aarch64", "i686", "ppc64le", "s390x", "armv7l"):
         assert _wheel_architecture(f"linux_{architecture}") == architecture
         assert _wheel_architecture(f"manylinux_2_28_{architecture}") == architecture
+
+
+@pytest.mark.parametrize(
+    "tag,minimums,violations",
+    [
+        # Binaries at or below the supported minimum.
+        ("macosx_14_0_arm64", {"a.dylib": (14, 0), "b.so": (13, 0)}, (False, [])),
+        # A newer major version, after bdist_wheel raised the tag to match it.
+        ("macosx_15_0_arm64", {"a.dylib": (15, 0)}, (True, ["a.dylib"])),
+        # A tag that promises an older macOS than the binaries support.
+        ("macosx_11_0_arm64", {"a.dylib": (14, 0)}, (False, ["a.dylib"])),
+        # A binary that declares no minimum cannot be shown to meet one.
+        ("macosx_14_0_arm64", {"a.dylib": None}, (False, ["a.dylib"])),
+    ],
+)
+def test_finds_what_stops_the_wheel_on_the_oldest_supported_macos(
+    tag, minimums, violations
+):
+    assert _macos_minimum_violations(tag, minimums) == violations
+
+
+# How otool -l prints the load command that holds a library's minimum macOS.
+_BUILD_VERSION = """\
+      cmd LC_BUILD_VERSION
+  cmdsize 32
+ platform 1
+    minos 14.0
+      sdk 15.5
+   ntools 1
+     tool 3
+  version 1167.5
+"""
+
+# The older command, which ld64 writes instead for targets below macOS 10.14.
+_VERSION_MIN = """\
+      cmd LC_VERSION_MIN_MACOSX
+  cmdsize 16
+  version 10.13
+      sdk 15.5
+"""
+
+
+@pytest.mark.parametrize(
+    "load_commands,minimum",
+    [
+        (_BUILD_VERSION, (14, 0)),
+        (_VERSION_MIN, (10, 13)),
+        # A file with neither command cannot show the minimum is met.
+        ("      cmd LC_SEGMENT_64\n", None),
+    ],
+)
+def test_reads_the_minimum_macos_of_a_mach_o_file(load_commands, minimum):
+    assert _mach_o_minimum(load_commands) == minimum
+
+
+@pytest.mark.parametrize(
+    "data,minimum",
+    [
+        # Newer compilers add a version to the architecture. The newest target counts.
+        (b"air64_v26-apple-macosx14.0.0\0air64_v26-apple-macosx15.0.0\0", (15, 0)),
+        # A Metal library that names no target cannot show the minimum is met.
+        (b"MTLB" + b"\0" * 64, None),
+    ],
+)
+def test_reads_the_minimum_macos_of_a_metal_library(data, minimum):
+    assert _metallib_minimum(data) == minimum
+
+
+# Fake wheel contents by path. A Mach-O file is its header followed by the minimum
+# macOS of each architecture, which the fake otool prints back.
+_THIN, _FAT64 = b"\xcf\xfa\xed\xfe", b"\xca\xfe\xba\xbf"
+_WHEEL = {
+    "executorch/lib/libexecutorch.dylib": _THIN + b"14.0",
+    "executorch/data/bin/flatc": _FAT64 + b"14.0 14.0",
+    "executorch/lib/mlx.metallib": b"MTLB air64-apple-macosx14.0.0",
+}
+
+
+@pytest.mark.parametrize(
+    "tag,files,error",
+    [
+        ("macosx_14_0_arm64", _WHEEL, None),
+        # The second architecture of a universal executable needs macOS 14.5.
+        (
+            "macosx_14_0_arm64",
+            {**_WHEEL, "executorch/data/bin/flatc": _FAT64 + b"14.0 14.5"},
+            r"flatc \(14\.5\)",
+        ),
+        (
+            "macosx_14_0_arm64",
+            {**_WHEEL, "executorch/lib/mlx.metallib": b"MTLB air64-apple-macosx14.5.0"},
+            r"mlx\.metallib \(14\.5\)",
+        ),
+        ("macosx_15_0_arm64", _WHEEL, "tagged for a macOS newer than 14.0"),
+        (
+            "macosx_14_0_arm64",
+            {**_WHEEL, "executorch/lib/libexecutorch.dylib": _THIN},
+            "otool could not read",
+        ),
+        (
+            "macosx_14_0_arm64",
+            {"executorch/lib/libexecutorch.dylib": b"not a Mach-O file"},
+            "contains no Mach-O files or Metal libraries",
+        ),
+    ],
+)
+def test_checks_every_binary_in_a_macos_wheel(tmp_path, monkeypatch, tag, files, error):
+    # The check runs on macOS after a wheel build, but all it reads is a zip file and
+    # what lipo and otool print, so here the wheel and both tools are fake.
+    wheel = tmp_path / f"executorch-1.0-cp311-cp311-{tag}.whl"
+    with zipfile.ZipFile(wheel, "w") as archive:
+        for name, data in files.items():
+            archive.writestr(name, data)
+
+    def run(command, **kwargs):
+        if command[0] == "lipo":
+            return subprocess.CompletedProcess(command, 0, "arm64\n", "")
+        minimums = Path(command[-1]).read_bytes()[4:].decode().split()
+        # Without -arch all, otool can print just one architecture of a universal file.
+        if "all" not in command:
+            minimums = minimums[:1]
+        printed = "".join(f"cmd LC_BUILD_VERSION\nminos {m}\n" for m in minimums)
+        return subprocess.CompletedProcess(command, 0 if minimums else 1, printed, "")
+
+    monkeypatch.setattr(sys, "platform", "darwin")
+    monkeypatch.setattr(test_shared_libraries, "_find_wheel_files", lambda: [wheel])
+    monkeypatch.setattr(test_shared_libraries, "_tool", lambda name: name)
+    monkeypatch.setattr(subprocess, "run", run)
+    if error is None:
+        test_shared_libraries.test_wheel_platform_tag()
+    else:
+        with pytest.raises(AssertionError, match=error):
+            test_shared_libraries.test_wheel_platform_tag()

.ci/scripts/wheel/envvar_macos.sh

@@ -14,3 +14,8 @@ source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh"
 # Apple SDK Objective-C framework headers (e.g. NSIntegerMax in NSObjCRuntime.h).
 export CC=/usr/bin/clang
 export CXX=/usr/bin/clang++
+
+# Pin Xcode so a runner image update cannot change the wheel unnoticed. Its SDK
+# decides which Core ML code is built in. 16.4 is the macOS 15 image's default. The
+# 26.0.1 pins in other workflows build PyTorch from source, which this build does not.
+export DEVELOPER_DIR=/Applications/Xcode_16.4.app/Contents/Developer

.ci/scripts/wheel/pre_build_script.sh

@@ -179,15 +179,18 @@ if [[ $UNAME_S == *"MINGW"* || $UNAME_S == *"MSYS"* ]]; then
   fi
 fi
 
-# Enable VGF in pybind wheel builds when the platform-specific build input is
-# available from pip.
+# Enable VGF in Linux pybind wheel builds when the platform-specific build input
+# is available from pip. Not on macOS: the VGF library's macOS wheels need macOS 15,
+# newer than the oldest macOS this wheel supports.
 if [[ "$UNAME_S" == "Linux" || "$UNAME_S" == "Darwin" ]]; then
   # The wheel rewrites recorded runtime paths after linking, which needs patchelf. It is declared
   # under build-system requires, but that is honoured only by a frontend with build isolation and
   # this path deliberately installs requirements itself, so without this the rewrite silently
   # does nothing and a CUDA wheel loses its route to the NVIDIA libraries beside it.
   python3 -m pip install patchelf || true
-  if python3 -m pip install -r \
+  if [[ "$UNAME_S" == "Darwin" ]]; then
+    echo "VGF needs a newer macOS than this wheel supports; building without VGF"
+  elif python3 -m pip install -r \
     "${GITHUB_WORKSPACE}/${REPOSITORY}/backends/arm/requirements-arm-vgf-runtime.txt"; then
     export EXECUTORCH_PYBIND_ENABLE_VGF=ON
     echo "EXECUTORCH_PYBIND_ENABLE_VGF=ON" >> "${GITHUB_ENV}"

.ci/scripts/wheel/test_shared_libraries.py

@@ -194,6 +194,8 @@
         b"\xfe\xed\xfa\xce",
         b"\xca\xfe\xba\xbe",
         b"\xbe\xba\xfe\xca",
+        b"\xca\xfe\xba\xbf",
+        b"\xbf\xba\xfe\xca",
     }
 )
 
@@ -769,6 +771,127 @@ def _assert_mach_o_architecture_matches(wheel: Path) -> None:
     print(f"\u2713 the wheel is tagged for the architecture it contains ({declared})")
 
 
+# The oldest macOS the wheel supports. The pybind preset sets it, and so does the
+# MACOSX_DEPLOYMENT_TARGET that the pytorch/test-infra wheel build exports.
+_MACOS_MINIMUM = (14, 0)
+
+
+def _macos_minimum_violations(tag: str, minimums: dict) -> tuple[bool, list]:
+    """Whether a macOS tag asks for more than the supported minimum, and which binaries
+    need more than the tag or the supported minimum allows.
+
+    Returned rather than asserted so the decision can be unit tested without a wheel.
+    `minimums` maps each binary to the minimum macOS it declares, or to None when it
+    declares none, which counts against it.
+    """
+    # In macosx_14_0_arm64 the two numbers after the platform name are the version.
+    tagged = tuple(int(part) for part in tag.split("_")[1:3])
+    allowed = min(tagged, _MACOS_MINIMUM)
+    return tagged > _MACOS_MINIMUM, sorted(
+        name
+        for name, minimum in minimums.items()
+        if minimum is None or minimum > allowed
+    )
+
+
+def _metallib_minimum(data: bytes) -> tuple[int, int] | None:
+    """The newest macOS that a kernel in a Metal library targets, or None if none says.
+
+    Each kernel keeps its target, such as air64-apple-macosx14.0.0, or
+    air64_v26-apple-macosx14.0.0 from newer compilers.
+    """
+    versions = re.findall(rb"air64[\w.]*-apple-macosx(\d+)\.(\d+)", data)
+    return max(((int(major), int(minor)) for major, minor in versions), default=None)
+
+
+def _mach_o_minimum(load_commands: str) -> tuple[int, int] | None:
+    """The newest minimum macOS in `otool -l` output, or None if it declares none.
+
+    A universal binary declares one for each architecture. Targets below macOS 10.14
+    store it as the version of the older LC_VERSION_MIN_MACOSX command.
+    """
+    versions = re.findall(
+        r"(?:LC_BUILD_VERSION.*?minos|LC_VERSION_MIN_MACOSX.*?version) (\d+)\.(\d+)",
+        load_commands,
+        re.DOTALL,
+    )
+    return max(((int(major), int(minor)) for major, minor in versions), default=None)
+
+
+def _assert_minimum_macos(wheel: Path) -> None:
+    """Fail if a macOS wheel needs a newer macOS than the project supports.
+
+    bdist_wheel raises the tag, and only warns, when a .so or .dylib needs a newer major
+    macOS, so pip on an older macOS quietly picks an older release. It ignores a newer
+    minor version, executables such as flatc, and Metal libraries, so those install and
+    then fail to load. Installing the wheel on the oldest supported macOS used to catch
+    both, and no build runner runs that macOS any more. This reads the minimum that each
+    Mach-O file and Metal library declares, so it cannot see a call to a newer macOS API
+    that skips its availability check, or code built for a newer macOS that was linked
+    into a library, since the linker records the library's own minimum and only warns.
+    """
+    otool = _tool("otool")
+    assert otool is not None, (
+        "otool is required to read the minimum macOS of the wheel's binaries, and it "
+        "was not found"
+    )
+    claimed = wheel.name.split("-")[-1].removesuffix(".whl")
+    floor = ".".join(str(part) for part in _MACOS_MINIMUM)
+
+    minimums, unreadable = {}, []
+    with tempfile.TemporaryDirectory() as unpacked:
+        with zipfile.ZipFile(wheel) as archive:
+            archive.extractall(unpacked)
+        root = Path(unpacked)
+        for path in sorted(root.rglob("*")):
+            if not path.is_file() or path.is_symlink():
+                continue
+            name = str(path.relative_to(root))
+            if path.suffix == ".metallib":
+                minimum = _metallib_minimum(path.read_bytes())
+            else:
+                with path.open("rb") as handle:
+                    if handle.read(4) not in _MACH_O_MAGIC:
+                        continue
+                result = subprocess.run(
+                    [otool, "-arch", "all", "-l", str(path)],
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                )
+                if result.returncode != 0:
+                    unreadable.append(f"{name}: {result.stderr.strip()[:120]}")
+                    continue
+                minimum = _mach_o_minimum(result.stdout)
+            minimums[name] = minimum
+
+    assert not unreadable, (
+        f"otool could not read these Mach-O files in {wheel.name}, so the minimum macOS "
+        f"check covered less than the wheel ships: {unreadable}"
+    )
+    assert minimums, (
+        f"the wheel {wheel.name} contains no Mach-O files or Metal libraries, so "
+        f"nothing was checked against macOS {floor}"
+    )
+    tag_too_new, too_new = _macos_minimum_violations(claimed, minimums)
+    assert not too_new, (
+        f"these files in {wheel.name} declare no minimum macOS, or a newer one than "
+        f"its tag or macOS {floor} allows: "
+        + ", ".join(
+            f"{name} ({'.'.join(map(str, minimums[name] or ())) or 'none'})"
+            for name in too_new
+        )
+    )
+    assert not tag_too_new, (
+        f"the wheel {wheel.name} is tagged for a macOS newer than {floor}, so pip on "
+        f"macOS {floor} will not install it"
+    )
+    print(
+        f"\u2713 the wheel's tag allows macOS {floor} and no Mach-O file or Metal "
+        "library in it declares a newer minimum"
+    )
+
+
 def _shipped_object_patterns() -> list[str]:
     """Filename patterns for every shipped loadable object.
 
@@ -2324,16 +2447,20 @@ def _tag_architectures_match(claimed: str, supported: str):
 def test_wheel_platform_tag() -> None:
     """The wheel's declared platform tag must name the architecture it was built for.
 
-    Only the architecture. auditwheel cannot certify a glibc baseline for this wheel:
-    it depends on torch without vendoring torch's libraries, so the contents reference
-    libtorch.so from outside any manylinux policy and auditwheel reports a plain
-    linux_<arch>. That is the expected answer for a torch-dependent wheel rather than
-    a defect, and the manylinux tag on the file comes from the build image. Asserting
-    the baseline here failed every correct wheel.
+    On Linux, only the architecture. auditwheel cannot certify a glibc baseline for
+    this wheel: it depends on torch without vendoring torch's libraries, so the contents
+    reference libtorch.so from outside any manylinux policy and auditwheel reports a
+    plain linux_<arch>. That is the expected answer for a torch-dependent wheel rather
+    than a defect, and the manylinux tag on the file comes from the build image.
+    Asserting the baseline here failed every correct wheel.
 
     The architecture is still worth checking, because a wheel labelled with the wrong
     one installs on machines it cannot run on at all, and that is a mistake this can
     actually catch.
+
+    On macOS the tag also names the oldest macOS the wheel installs on. That is
+    checked too, because a build that moves it only warns, and no CI job runs that
+    macOS any more.
     """
     wheels = _find_wheel_files()
     if not wheels:
@@ -2353,6 +2480,7 @@ def test_wheel_platform_tag() -> None:
     # correct wheel to fail the job.
     if sys.platform == "darwin":
         _assert_mach_o_architecture_matches(wheels[-1])
+        _assert_minimum_macos(wheels[-1])
         return
     if _WINDOWS:
         _assert_pe_architecture_matches(wheels[-1])

.github/workflows/build-presets.yml

@@ -27,7 +27,7 @@ jobs:
       default-packages: ""
       job-name: build
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      runner: macos-14-xlarge
+      runner: macos-15-xlarge
       python-version: 3.12
       submodules: recursive
       timeout: 90