diff --git a/.github/workflows/exporter-build-check.yml b/.github/workflows/exporter-build-check.yml new file mode 100644 index 00000000..45b1d9fc --- /dev/null +++ b/.github/workflows/exporter-build-check.yml @@ -0,0 +1,145 @@ +# Builds the exporter on Windows and proves the binary runs, on pull requests. +# +# **The release workflow only fires on `release: published`**, so until this existed the Windows +# binary was first built at the moment it was published, and the first person to run it was a user. +# That is how 1.2.0 shipped an .exe that died with `OpenTagViewer.exe parou de funcionar` - the +# process killed by Windows, no traceback, nothing in any log, because a native fault does not +# raise. Nothing in CI had ever started it. +# +# This does not upload anything and is not part of releasing. It exists to fail a pull request +# rather than a download. +# +# **Windows only, deliberately.** It is the platform whose build had no coverage at all before a +# release, and the one that broke. Adding `macos-14` or `ubuntu-22.04` to the matrix is a two-line +# change if the same happens elsewhere - the steps below are otherwise platform-agnostic - but +# every entry costs a PyInstaller build on every pull request, so they are not there by default. + +name: Exporter Build Check + +on: + pull_request: + paths: + - "python/**" + - ".github/workflows/exporter-build-check.yml" + push: + branches: [ "main" ] + paths: + - "python/**" + - ".github/workflows/exporter-build-check.yml" + workflow_dispatch: + +env: + PYINSTALLER_PYTHON_VER: 3.13 + +jobs: + windows: + name: Build and start the Windows binary + runs-on: windows-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYINSTALLER_PYTHON_VER }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYINSTALLER_PYTHON_VER }} + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + # --frozen: exactly what uv.lock records. A build check that resolved something else would be + # checking a bundle nobody is going to ship. + # + # `--group build` is where PyInstaller lives, and without it this fails with "Failed to spawn: + # pyinstaller / program not found" - which is what the first run of this workflow did. Copied + # from the release workflow's install step for the same reason its PyInstaller call is copied: + # a build check that installs a different set is checking a different bundle. + - name: Install dependencies + working-directory: ./python + run: uv sync --frozen --no-default-groups --group build + + # **Before freezing anything**, because it separates two very different faults that look + # identical from outside. The frozen binary dies inside `emu_start` after importing unicorn, + # constructing it and mapping memory - so the native library loads and the bundle is fine. + # What is not yet known is whether *executing* emulated ARM64 works on Windows at all. + # + # If this passes and the frozen one fails, it is PyInstaller. If both fail, unicorn cannot run + # here and local Anisette can never work on Windows - which is a product decision, not a build + # fix, because the sign-in would have to use a remote Anisette server instead. + # + # `continue-on-error` so the answer is recorded even when it is "no": this step is an + # experiment, and a red X here would hide the frozen result below it. + - name: Does the emulator run under plain Python on this machine + continue-on-error: true + working-directory: ./python + run: | + uv run --frozen python -c "import unicorn, sys; print('unicorn', unicorn.__file__, flush=True); uc = unicorn.Uc(unicorn.UC_ARCH_ARM64, unicorn.UC_MODE_ARM); print('constructed', flush=True); uc.mem_map(0x1000, 0x1000); uc.mem_write(0x1000, bytes.fromhex('400580d2')); print('written', flush=True); uc.emu_start(0x1000, 0x1004); from unicorn.arm64_const import UC_ARM64_REG_X0; print('x0 =', uc.reg_read(UC_ARM64_REG_X0), flush=True)" + + # Kept identical to the release workflow's Windows step on purpose. If the two drift, this + # stops testing what gets published, which is worse than not testing at all because it reads + # as coverage. + # + # --collect-all unicorn is not optional and its absence is invisible until somebody runs the + # result: unicorn loads its native library through ctypes, so PyInstaller's analysis never sees + # it. --collect-data covers the pinned root certificates FindMy.py keeps as files beside its + # code, which a build once omitted and which failed several minutes into a real sign-in. + - name: Run PyInstaller + working-directory: ./python + run: | + uv run --frozen pyinstaller ` + --windowed ` + --name "OpenTagViewer" ` + --icon=OpenTagViewer.ico ` + --collect-all unicorn ` + --collect-data findmy ` + --collect-data anisette ` + --noconfirm ` + ./exporter/wizard.py + + # Two checks, and the second is the one that would have caught the crash this workflow exists + # for. `--version` proves the bundle can import everything it was built with, Tk included. + # `--self-test` reaches the data files *and* runs four bytes of ARM64 through the emulator, + # which is the part that is native, the part PyInstaller cannot see, and the part that killed + # the process on a user's machine rather than raising anything a log could hold. + # + # PowerShell rather than bash, only so the exit code survives. bash reports a process killed + # by Windows as 127, which is its own "command not found" and says nothing; the real value is + # an NTSTATUS and it names the mechanism. 0xC0000005 is an access violation, 0xC0000409 is a + # security check - Control Flow Guard or a stack cookie - and those point at very different + # fixes for a library that JITs its own code the way unicorn does. + # **Control Flow Guard has to come off, or the program cannot sign in.** unicorn compiles + # ARM64 into native code at runtime and jumps to it; CFG checks every indirect call against + # targets the linker knew about, and code that did not exist at link time is not among them. + # The process is then killed by __fastfail with no exception and no traceback - observed here + # as exit `0xC0000409` from `emu_start`, and reported by a user as "parou de funcionar". + # + # PyInstaller's bootloader is built with CFG on and the flag is process-wide, so no packaging + # option avoids it. See the script for why giving up the mitigation is the right trade. + - name: Turn off Control Flow Guard, so the emulator can run + run: python scripts/clear_windows_cfg.py python/dist/OpenTagViewer/OpenTagViewer.exe + + # **Asserted on the output, not on the exit code**, because the exit code cannot be trusted + # here and trusting it already produced a green run over a dead binary. A process Windows + # kills leaves `$LASTEXITCODE` unset, `exit $null` is `exit 0`, and the check passed while the + # self-test was still dying at `emu_start`. A build check that goes green over the failure it + # was written for is worse than not having one, because it reads as coverage. + # + # So success is the line the self-test prints when it finishes, and nothing else counts. + - name: Start the binary and run its self-test + working-directory: ./python/dist + shell: pwsh + run: | + ./OpenTagViewer/OpenTagViewer --version + + $output = (& ./OpenTagViewer/OpenTagViewer --self-test 2>&1 | Out-String) + $code = $LASTEXITCODE + + Write-Host $output + Write-Host ("self-test exit code: '{0}'" -f $code) + + if ($output -notmatch 'self-test passed') { + Write-Host "The self-test did not report success. The last line above is the call that" + Write-Host "killed it - a native fault raises nothing, so there is no traceback to find." + exit 1 + } diff --git a/.github/workflows/macos-exporter-python.yml b/.github/workflows/macos-exporter-python.yml index 4c2ebb8f..cd9a14fd 100644 --- a/.github/workflows/macos-exporter-python.yml +++ b/.github/workflows/macos-exporter-python.yml @@ -262,6 +262,23 @@ jobs: --noconfirm ` ./exporter/wizard.py + # **Without this the Windows build cannot sign in**, and fails in the way that is hardest to + # report: the process vanishes, with no exception, no traceback and nothing in the log. + # + # unicorn compiles ARM64 into native code at runtime and jumps to it, which is how Apple's ADI + # library is emulated and therefore how a login happens without a third-party Anisette server. + # Control Flow Guard checks every indirect call against targets the linker knew about, and code + # that did not exist at link time is not among them, so the jump is refused and __fastfail + # terminates the process. PyInstaller's bootloader is built with CFG on and the flag applies to + # the whole process, so nothing about how the Python side is packaged avoids it. + # + # Reported by a user as "OpenTagViewer.exe parou de funcionar" on 1.2.0, and reproduced in + # exporter-build-check.yml as exit 0xC0000409 from `emu_start`. The script explains why giving + # up the mitigation is the right trade for this program. + - name: Turn off Control Flow Guard (Windows), so the emulator can run + if: matrix.os == 'windows' + run: python scripts/clear_windows_cfg.py python/dist/OpenTagViewer/OpenTagViewer.exe + # No --windowed on Linux: it means "no console window" on the platforms that have that # concept, and there is no bundle format here for it to produce either way. # --collect-all unicorn is not optional, and its absence is invisible until somebody runs the diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7bece2a2..2687c792 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -570,6 +570,7 @@ run regardless: each AES entry carries a fresh random salt. | `build-release.yml` | on release | Translation check, JVM tests, Chaquopy bridge tests, release APK | | `macos-scripts-python.yml` | `python/**` changes | Exporter and shared-package tests across Python 3.10–3.13 on macOS 14 | | `macos-exporter-python.yml` | on release | Tag/version check, exporter tests, and the PyInstaller build for macOS (both architectures), Windows and Linux | +| `exporter-build-check.yml` | PR and push to `main`, `python/**` | Builds the Windows binary and starts it. The release workflow above only runs on `release: published`, so without this a broken bundle is first run by whoever downloads it | | `update-contributors.yml` | weekly | Regenerates the contributor list on the Information page, opens a PR if it changed | | `check-adi-libraries.yml` | weekly | Checks Apple's ADI libraries still match what is checked in, opens an issue if they drifted | diff --git a/python/exporter/version.py b/python/exporter/version.py index b08a8a11..64f31ae0 100644 --- a/python/exporter/version.py +++ b/python/exporter/version.py @@ -19,7 +19,7 @@ from __future__ import annotations -VERSION = "1.2.0" +VERSION = "1.2.1" APP_TITLE = f"OpenTagViewer AirTag Exporter {VERSION}" diff --git a/python/exporter/wizard.py b/python/exporter/wizard.py index e76de52c..0e45ef51 100644 --- a/python/exporter/wizard.py +++ b/python/exporter/wizard.py @@ -994,22 +994,111 @@ def self_test() -> int: # Reads every `.crt` beside the library's code and checks each against its own fingerprint, # so this proves the files are both present and intact. roots = PinnedRoots.bundled().by_version - print(f"pinned roots: {len(roots)} ({', '.join(str(v) for v in sorted(roots))})") + _say(f"pinned roots: {len(roots)} ({', '.join(str(v) for v in sorted(roots))})") # A namespace package would have no __file__, and a build that produced one would be broken in # a way worth reporting rather than crashing on. anisette_module = sys.modules["anisette"].__file__ anisette_root = Path(anisette_module).parent / "apple-root.pem" if anisette_module else None found = anisette_root is not None and anisette_root.is_file() - print(f"anisette root: {'present' if found else 'MISSING'}") + _say(f"anisette root: {'present' if found else 'MISSING'}") if not roots or not found: return 1 - print("self-test passed") + if not _emulator_runs(): + return 1 + + _say("self-test passed") return 0 +def _say(line: str) -> None: + """ + Print, and flush before the next thing can kill the process. + + **A native fault loses buffered output**, and stdout is block-buffered the moment it is a pipe + rather than a terminal - which it always is in CI. So the first run of this on Windows died + with no output at all: every line was still sitting in the buffer when the process was killed, + and a self-test that cannot say how far it got is only marginally better than no self-test. + """ + print(line, flush=True) + + +def _emulator_runs() -> bool: + """ + Actually run the CPU emulator, rather than checking its file is present. + + **The two are not the same check, and the difference is a hard crash.** `unicorn` loads its + native library through ctypes at runtime, so PyInstaller never sees it and `--collect-all + unicorn` is what puts the file in the bundle. That makes the file *present*; it says nothing + about whether the library loads and executes on the machine that ends up running it. + + A Windows user on 1.2.0 reported `OpenTagViewer.exe parou de funcionar` - the process killed + by Windows, no Python traceback, nothing in any log, because a native fault does not raise. The + checks above would all have passed: the data files were there. Nothing exercised the code. + + So this emulates four bytes of ARM64 and reads the register back. Local, instant, no network + and no account - and it fails a build rather than a user's machine, which is the entire point. + Anisette needs this to work, because emulating Apple's ADI library is how a sign-in happens + without a third-party server. + + **Reported one call at a time**, because a native fault produces no exception and no + traceback - so the only thing that says where it died is the last line that made it out. + Each stage announces itself before it runs, and the failing call is whichever one has no + answer after it. That is what turned "the Windows build dies somewhere" into a line number. + """ + state: dict = {} + + for name, step in ( + ("import", _emu_import), + ("construct", _emu_construct), + ("mem_map", _emu_map), + ("mem_write", _emu_write), + ("emu_start", _emu_run), + ): + _say(f"cpu emulator: {name} …") + try: + step(state) + except Exception as e: # noqa: BLE001 - a broken bundle, not a bug worth raising + _say(f"cpu emulator: {name} FAILED ({type(e).__name__}: {e})") + return False + + answer = state["answer"] + _say(f"cpu emulator: ran ARM64 and read back {answer}") + + return answer == 42 + + +def _emu_import(state: dict) -> None: + import unicorn # noqa: PLC0415 + from unicorn import arm64_const # noqa: PLC0415 + + state["unicorn"] = unicorn + state["arm64"] = arm64_const + # The path matters: a bundle can carry the Python package and miss the native library beside + # it, and then this is the last line before the process disappears. + _say(f"cpu emulator: loaded {getattr(unicorn, '__file__', '?')}") + + +def _emu_construct(state: dict) -> None: + unicorn = state["unicorn"] + state["uc"] = unicorn.Uc(unicorn.UC_ARCH_ARM64, unicorn.UC_MODE_ARM) + + +def _emu_map(state: dict) -> None: + state["uc"].mem_map(0x1000, 0x1000) + + +def _emu_write(state: dict) -> None: + state["uc"].mem_write(0x1000, bytes.fromhex("400580d2")) # movz x0, #42 + + +def _emu_run(state: dict) -> None: + state["uc"].emu_start(0x1000, 0x1004) + state["answer"] = state["uc"].reg_read(state["arm64"].UC_ARM64_REG_X0) + + if __name__ == "__main__": if "--self-test" in sys.argv[1:]: import anisette # noqa: F401, PLC0415 - imported for its path, above diff --git a/scripts/clear_windows_cfg.py b/scripts/clear_windows_cfg.py new file mode 100644 index 00000000..8eaa6545 --- /dev/null +++ b/scripts/clear_windows_cfg.py @@ -0,0 +1,120 @@ +""" +Turn off Control Flow Guard on a built Windows executable, so a JIT can run inside it. + +**Why this exists.** The exporter signs in by emulating Apple's ADI library, which `unicorn` does +by compiling ARM64 into native code at runtime and jumping to it. Control Flow Guard is a Windows +mitigation that checks every indirect call against a table of targets the linker knew about, and +code that did not exist at link time is not in that table. So the jump is rejected and the process +is terminated by `__fastfail` - **no exception, no traceback, nothing in any log**, because the +whole point of the mechanism is to stop the process before anything can handle it. + +That is exactly what a user reported as `OpenTagViewer.exe parou de funcionar`, and what CI +reproduced: `import`, construct, `mem_map` and `mem_write` all succeed, and `emu_start` returns +exit code `0xC0000409`. + +PyInstaller's bootloader is built with CFG enabled and the flag applies to the whole process, so +nothing about how the Python side is packaged can avoid it. The flag lives in one bit of the PE +header, and clearing it after the build is the smallest change that makes the program work. + +**This is a real trade, not a formality.** CFG is a genuine hardening measure and this gives it up +for the whole executable. It is given up because the alternative is an exporter that cannot sign in +on Windows at all: the emulator is not an optional component, it is how the login works without +handing the exchange to a third-party Anisette server. Every program that JITs - browsers, .NET, +every language runtime with a tracing compiler - either disables CFG or registers its targets with +`SetProcessValidCallTargets`, and the second is not reachable from here. + +Usage: + + python scripts/clear_windows_cfg.py path/to/OpenTagViewer.exe +""" + +from __future__ import annotations + +import struct +import sys +from pathlib import Path + +# IMAGE_DLLCHARACTERISTICS_GUARD_CF. One bit, and the difference between a program that runs and +# one that is killed on its first jump into compiled code. +GUARD_CF = 0x4000 + +PE32 = 0x10B +PE32_PLUS = 0x20B + +# Offset of DllCharacteristics within the optional header. The same for PE32 and PE32+: the two +# differ earlier - PE32+ drops BaseOfData and widens ImageBase to eight bytes - and the layouts +# realign at SectionAlignment, well before this field. +DLL_CHARACTERISTICS_OFFSET = 0x46 + + +class NotAPortableExecutable(Exception): + """The file is not a PE image, so there is no flag here to change.""" + + +def clear_guard_cf(path: Path) -> bool: + """ + Clear the CFG bit in a PE image, in place. + + :returns: Whether the bit was set to begin with, so a caller can tell "turned it off" from + "there was nothing to turn off" - which are different, and only the first is expected. + :raises NotAPortableExecutable: If the file does not have the headers this needs. Raised + rather than ignored: silently doing nothing to the wrong file would produce a build that + looks patched and still dies. + """ + data = bytearray(path.read_bytes()) + + if data[:2] != b"MZ": + msg = f"{path} does not start with MZ, so it is not a PE image." + raise NotAPortableExecutable(msg) + + pe_offset = struct.unpack_from(" int: + if len(argv) != 1: + print(__doc__) + return 2 + + path = Path(argv[0]) + + try: + cleared = clear_guard_cf(path) + except (OSError, NotAPortableExecutable) as e: + print(f"Could not clear Control Flow Guard on {path}: {e}", file=sys.stderr) + return 1 + + if cleared: + print(f"Control Flow Guard turned off on {path}") + else: + # Worth saying rather than passing quietly. It means either the toolchain stopped setting + # it - in which case this script has become unnecessary - or the wrong file was patched, + # and those want very different follow-ups. + print(f"Control Flow Guard was already off on {path}; nothing changed") + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test/test_clear_windows_cfg.py b/scripts/test/test_clear_windows_cfg.py new file mode 100644 index 00000000..9c1806b5 --- /dev/null +++ b/scripts/test/test_clear_windows_cfg.py @@ -0,0 +1,124 @@ +""" +Clearing the Control Flow Guard bit from a built Windows executable. + +**Tested on a synthetic PE, because the real one only exists on a Windows runner.** The header +layout is fixed by the PE specification, so a handful of bytes in the right places exercises the +same arithmetic the real file does - and the alternative is a script whose only test is a CI job +on another operating system, which is how a patch that silently does nothing gets shipped. + +What matters here is not only that the bit is cleared. It is that the wrong file is *refused*: +a build step that quietly succeeds on something it did not understand produces a binary that looks +patched and still dies on a user's machine, with the one symptom that leaves no traceback. +""" + +from __future__ import annotations + +import struct +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from clear_windows_cfg import ( # noqa: E402 + GUARD_CF, + PE32, + PE32_PLUS, + NotAPortableExecutable, + clear_guard_cf, +) + +PE_OFFSET = 0x80 +OPTIONAL = PE_OFFSET + 4 + 20 +FIELD = OPTIONAL + 0x46 + + +def build_pe(characteristics: int, *, magic: int = PE32_PLUS) -> bytes: + """The smallest thing that is shaped like a PE image as far as this script is concerned.""" + data = bytearray(FIELD + 2) + data[0:2] = b"MZ" + struct.pack_into(" int: + return struct.unpack_from(" Path: + path = tmp_path / "OpenTagViewer.exe" + path.write_bytes(build_pe(characteristics, **kwargs)) + return path + + return write + + +class TestClearingTheBit: + def test_it_turns_control_flow_guard_off(self, executable): + path = executable(GUARD_CF) + + assert clear_guard_cf(path) is True + assert characteristics_of(path) & GUARD_CF == 0 + + def test_it_leaves_every_other_flag_alone(self, executable): + # The field carries ASLR, DEP and the rest beside it. Clearing the wrong bits would turn a + # mitigation problem into a different mitigation problem, on a binary handed to users. + others = 0x0040 | 0x0100 | 0x8000 # DYNAMIC_BASE, NX_COMPAT, TERMINAL_SERVER_AWARE + path = executable(GUARD_CF | others) + + clear_guard_cf(path) + + assert characteristics_of(path) == others + + def test_a_file_that_never_had_it_reports_no_change(self, executable): + # Distinct from success, and the caller says so: it means either the toolchain stopped + # setting the bit, or the wrong file was patched. Those want different follow-ups. + path = executable(0x0100) + + assert clear_guard_cf(path) is False + assert characteristics_of(path) == 0x0100 + + @pytest.mark.parametrize("magic", [PE32, PE32_PLUS]) + def test_both_header_widths_are_handled(self, executable, magic): + # PE32 and PE32+ differ earlier in the optional header and realign well before this field. + path = executable(GUARD_CF, magic=magic) + + assert clear_guard_cf(path) is True + + +class TestRefusingTheWrongFile: + """ + Each of these would otherwise be a build step that appears to work. + + That is the failure worth guarding: the exe reaches a user unpatched, and the symptom is a + process that vanishes without an exception, which is the hardest thing there is to report. + """ + + def test_something_that_is_not_an_executable(self, tmp_path): + path = tmp_path / "notes.txt" + path.write_bytes(b"this is not a program") + + with pytest.raises(NotAPortableExecutable, match="does not start with MZ"): + clear_guard_cf(path) + + def test_a_dos_header_pointing_at_nothing(self, tmp_path): + data = bytearray(build_pe(GUARD_CF)) + data[PE_OFFSET:PE_OFFSET + 4] = b"junk" + path = tmp_path / "OpenTagViewer.exe" + path.write_bytes(bytes(data)) + + with pytest.raises(NotAPortableExecutable, match="no PE signature"): + clear_guard_cf(path) + + def test_an_optional_header_of_an_unknown_shape(self, executable): + path = executable(GUARD_CF, magic=0x1234) + + with pytest.raises(NotAPortableExecutable, match="neither PE32 nor PE32"): + clear_guard_cf(path)