From 68a8000998963a3d0ab5b67ae36343d506ed70a0 Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 08:31:36 +0200 Subject: [PATCH 1/9] Bump the exporter to 1.2.1 1.2.0 went out at 12:29 UTC on the 16th and the fix for the crash in #89 merged at 14:08, so the current release is the one that cannot get past it - and a second user has now hit it, from a downloaded build rather than a clone. Their error is byte-for-byte identical to the first: "element claims 109 bytes, 61 remain", on a different account and a different operating system. That rules out the reading this had when only one report existed. It is not one unlucky item in sixty-one; a structure of exactly that shape sits in the ProtectedCloudStorage view of unrelated accounts, and every user who has one loses the whole export to it. Nothing new is being shipped - the fix has been on main since the 16th and is verified again here against the pin main now carries, which has moved since. Three changes reach the exporter since 1.2.0: that fix, one FindMy.py pin for the whole repository, and a secret file being usable on Windows. 441 tests, flake8 clean, and `release_version.py --kind exporter --tag exporter-v1.2.1` passes. --- python/exporter/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/exporter/version.py b/python/exporter/version.py index b08a8a11..64f31ae0 100644 --- a/python/exporter/version.py +++ b/python/exporter/version.py @@ -19,7 +19,7 @@ from __future__ import annotations -VERSION = "1.2.0" +VERSION = "1.2.1" APP_TITLE = f"OpenTagViewer AirTag Exporter {VERSION}" From 2dfbd490c7da38b68b0fbb9a11343e4949e08b78 Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 08:38:14 +0200 Subject: [PATCH 2/9] Build the Windows binary on pull requests, and make the self-test run native code A Windows user on 1.2.0 got `OpenTagViewer.exe parou de funcionar` - the process killed by Windows, no traceback, nothing in any log, because a native fault does not raise. Two gaps let that reach them. **Nothing ever built the Windows binary before a release.** The exporter workflow fires on `release: published` and nothing else, so the first run of any .exe was by whoever downloaded it. `exporter-build-check.yml` builds it on pull requests touching python/ and starts it. It uploads nothing and is not part of releasing; its whole job is to fail a PR instead of a download. Windows only for now, because it is the platform that had no pre-release coverage and the one that broke - the steps are otherwise platform-agnostic and the matrix is two lines to widen. **And the self-test proved files were present without running any code.** It checked the pinned roots and anisette's certificate, both of which would have passed here. unicorn loads its native library through ctypes, so `--collect-all unicorn` puts the file in the bundle and nothing establishes that it loads or executes - which is exactly the shape of a hard crash with no Python error to catch. So it now emulates four bytes of ARM64 and reads the register back. Local, instant, no network and no account, and it fails a build rather than a machine. Anisette needs that path to work, since emulating Apple's ADI library is how a sign-in happens without a third-party server. Not a fix: what crashed on that user's machine is still unknown, and the emulator is a hypothesis rather than a diagnosis. This is what turns the next one into a red build. --- .github/workflows/exporter-build-check.yml | 88 ++++++++++++++++++++++ CONTRIBUTING.md | 1 + python/exporter/wizard.py | 39 ++++++++++ 3 files changed, 128 insertions(+) create mode 100644 .github/workflows/exporter-build-check.yml diff --git a/.github/workflows/exporter-build-check.yml b/.github/workflows/exporter-build-check.yml new file mode 100644 index 00000000..b0c7602f --- /dev/null +++ b/.github/workflows/exporter-build-check.yml @@ -0,0 +1,88 @@ +# Builds the exporter on Windows and proves the binary runs, on pull requests. +# +# **The release workflow only fires on `release: published`**, so until this existed the Windows +# binary was first built at the moment it was published, and the first person to run it was a user. +# That is how 1.2.0 shipped an .exe that died with `OpenTagViewer.exe parou de funcionar` - the +# process killed by Windows, no traceback, nothing in any log, because a native fault does not +# raise. Nothing in CI had ever started it. +# +# This does not upload anything and is not part of releasing. It exists to fail a pull request +# rather than a download. +# +# **Windows only, deliberately.** It is the platform whose build had no coverage at all before a +# release, and the one that broke. Adding `macos-14` or `ubuntu-22.04` to the matrix is a two-line +# change if the same happens elsewhere - the steps below are otherwise platform-agnostic - but +# every entry costs a PyInstaller build on every pull request, so they are not there by default. + +name: Exporter Build Check + +on: + pull_request: + paths: + - "python/**" + - ".github/workflows/exporter-build-check.yml" + push: + branches: [ "main" ] + paths: + - "python/**" + - ".github/workflows/exporter-build-check.yml" + workflow_dispatch: + +env: + PYINSTALLER_PYTHON_VER: 3.13 + +jobs: + windows: + name: Build and start the Windows binary + runs-on: windows-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYINSTALLER_PYTHON_VER }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYINSTALLER_PYTHON_VER }} + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + # --frozen: exactly what uv.lock records. A build check that resolved something else would be + # checking a bundle nobody is going to ship. + - name: Install dependencies + working-directory: ./python + run: uv sync --frozen + + # Kept identical to the release workflow's Windows step on purpose. If the two drift, this + # stops testing what gets published, which is worse than not testing at all because it reads + # as coverage. + # + # --collect-all unicorn is not optional and its absence is invisible until somebody runs the + # result: unicorn loads its native library through ctypes, so PyInstaller's analysis never sees + # it. --collect-data covers the pinned root certificates FindMy.py keeps as files beside its + # code, which a build once omitted and which failed several minutes into a real sign-in. + - name: Run PyInstaller + working-directory: ./python + run: | + uv run --frozen pyinstaller ` + --windowed ` + --name "OpenTagViewer" ` + --icon=OpenTagViewer.ico ` + --collect-all unicorn ` + --collect-data findmy ` + --collect-data anisette ` + --noconfirm ` + ./exporter/wizard.py + + # Two checks, and the second is the one that would have caught the crash this workflow exists + # for. `--version` proves the bundle can import everything it was built with, Tk included. + # `--self-test` reaches the data files *and* runs four bytes of ARM64 through the emulator, + # which is the part that is native, the part PyInstaller cannot see, and the part that killed + # the process on a user's machine rather than raising anything a log could hold. + - name: Start the binary and run its self-test + working-directory: ./python/dist + shell: bash + run: | + ./OpenTagViewer/OpenTagViewer --version + ./OpenTagViewer/OpenTagViewer --self-test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7bece2a2..2687c792 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -570,6 +570,7 @@ run regardless: each AES entry carries a fresh random salt. | `build-release.yml` | on release | Translation check, JVM tests, Chaquopy bridge tests, release APK | | `macos-scripts-python.yml` | `python/**` changes | Exporter and shared-package tests across Python 3.10–3.13 on macOS 14 | | `macos-exporter-python.yml` | on release | Tag/version check, exporter tests, and the PyInstaller build for macOS (both architectures), Windows and Linux | +| `exporter-build-check.yml` | PR and push to `main`, `python/**` | Builds the Windows binary and starts it. The release workflow above only runs on `release: published`, so without this a broken bundle is first run by whoever downloads it | | `update-contributors.yml` | weekly | Regenerates the contributor list on the Information page, opens a PR if it changed | | `check-adi-libraries.yml` | weekly | Checks Apple's ADI libraries still match what is checked in, opens an issue if they drifted | diff --git a/python/exporter/wizard.py b/python/exporter/wizard.py index e76de52c..4d09b291 100644 --- a/python/exporter/wizard.py +++ b/python/exporter/wizard.py @@ -1006,10 +1006,49 @@ def self_test() -> int: if not roots or not found: return 1 + if not _emulator_runs(): + return 1 + print("self-test passed") return 0 +def _emulator_runs() -> bool: + """ + Actually run the CPU emulator, rather than checking its file is present. + + **The two are not the same check, and the difference is a hard crash.** `unicorn` loads its + native library through ctypes at runtime, so PyInstaller never sees it and `--collect-all + unicorn` is what puts the file in the bundle. That makes the file *present*; it says nothing + about whether the library loads and executes on the machine that ends up running it. + + A Windows user on 1.2.0 reported `OpenTagViewer.exe parou de funcionar` - the process killed + by Windows, no Python traceback, nothing in any log, because a native fault does not raise. The + checks above would all have passed: the data files were there. Nothing exercised the code. + + So this emulates four bytes of ARM64 and reads the register back. Local, instant, no network + and no account - and it fails a build rather than a user's machine, which is the entire point. + Anisette needs this to work, because emulating Apple's ADI library is how a sign-in happens + without a third-party server. + """ + try: + from unicorn import UC_ARCH_ARM64, UC_MODE_ARM, Uc # noqa: PLC0415 + from unicorn.arm64_const import UC_ARM64_REG_X0 # noqa: PLC0415 + + emulator = Uc(UC_ARCH_ARM64, UC_MODE_ARM) + emulator.mem_map(0x1000, 0x1000) + emulator.mem_write(0x1000, bytes.fromhex("400580d2")) # movz x0, #42 + emulator.emu_start(0x1000, 0x1004) + answer = emulator.reg_read(UC_ARM64_REG_X0) + except Exception as e: # noqa: BLE001 - any failure here is a broken bundle, not a bug to raise + print(f"cpu emulator: FAILED ({type(e).__name__}: {e})") + return False + + print(f"cpu emulator: ran ARM64 and read back {answer}") + + return answer == 42 + + if __name__ == "__main__": if "--self-test" in sys.argv[1:]: import anisette # noqa: F401, PLC0415 - imported for its path, above From 3c4dab808c46ac8b23300c0bf620f602eb673efa Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 08:43:30 +0200 Subject: [PATCH 3/9] Install the group PyInstaller is actually in The first run of this workflow failed with "Failed to spawn: pyinstaller / program not found". `uv sync --frozen` installs the default groups, and PyInstaller is in `build` - so the check built nothing and would have gone green the moment anybody made it non-blocking. Taken from the release workflow's install step verbatim, for the same reason its PyInstaller invocation is: a build check that installs a different set is checking a different bundle. --- .github/workflows/exporter-build-check.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/exporter-build-check.yml b/.github/workflows/exporter-build-check.yml index b0c7602f..6b4332e3 100644 --- a/.github/workflows/exporter-build-check.yml +++ b/.github/workflows/exporter-build-check.yml @@ -50,9 +50,14 @@ jobs: # --frozen: exactly what uv.lock records. A build check that resolved something else would be # checking a bundle nobody is going to ship. + # + # `--group build` is where PyInstaller lives, and without it this fails with "Failed to spawn: + # pyinstaller / program not found" - which is what the first run of this workflow did. Copied + # from the release workflow's install step for the same reason its PyInstaller call is copied: + # a build check that installs a different set is checking a different bundle. - name: Install dependencies working-directory: ./python - run: uv sync --frozen + run: uv sync --frozen --no-default-groups --group build # Kept identical to the release workflow's Windows step on purpose. If the two drift, this # stops testing what gets published, which is worse than not testing at all because it reads From 161e55302a7212a8dc2e0ba11c216ddda88db1b7 Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 08:48:32 +0200 Subject: [PATCH 4/9] Flush each self-test line, so a native death says how far it got The first Windows run produced no output at all and exit 127. Every line was still in the buffer when the process was killed: stdout is block-buffered the moment it is a pipe rather than a terminal, which it always is in CI, and a native fault never flushes. That is the difference between "the self-test failed" and "it failed at the emulator", and only the second is worth having. `--version` looked fine for the same reason it always did - it exits normally, so its buffer is flushed on the way out. --- python/exporter/wizard.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/python/exporter/wizard.py b/python/exporter/wizard.py index 4d09b291..1501a5a9 100644 --- a/python/exporter/wizard.py +++ b/python/exporter/wizard.py @@ -994,14 +994,14 @@ def self_test() -> int: # Reads every `.crt` beside the library's code and checks each against its own fingerprint, # so this proves the files are both present and intact. roots = PinnedRoots.bundled().by_version - print(f"pinned roots: {len(roots)} ({', '.join(str(v) for v in sorted(roots))})") + _say(f"pinned roots: {len(roots)} ({', '.join(str(v) for v in sorted(roots))})") # A namespace package would have no __file__, and a build that produced one would be broken in # a way worth reporting rather than crashing on. anisette_module = sys.modules["anisette"].__file__ anisette_root = Path(anisette_module).parent / "apple-root.pem" if anisette_module else None found = anisette_root is not None and anisette_root.is_file() - print(f"anisette root: {'present' if found else 'MISSING'}") + _say(f"anisette root: {'present' if found else 'MISSING'}") if not roots or not found: return 1 @@ -1009,10 +1009,22 @@ def self_test() -> int: if not _emulator_runs(): return 1 - print("self-test passed") + _say("self-test passed") return 0 +def _say(line: str) -> None: + """ + Print, and flush before the next thing can kill the process. + + **A native fault loses buffered output**, and stdout is block-buffered the moment it is a pipe + rather than a terminal - which it always is in CI. So the first run of this on Windows died + with no output at all: every line was still sitting in the buffer when the process was killed, + and a self-test that cannot say how far it got is only marginally better than no self-test. + """ + print(line, flush=True) + + def _emulator_runs() -> bool: """ Actually run the CPU emulator, rather than checking its file is present. @@ -1041,10 +1053,10 @@ def _emulator_runs() -> bool: emulator.emu_start(0x1000, 0x1004) answer = emulator.reg_read(UC_ARM64_REG_X0) except Exception as e: # noqa: BLE001 - any failure here is a broken bundle, not a bug to raise - print(f"cpu emulator: FAILED ({type(e).__name__}: {e})") + _say(f"cpu emulator: FAILED ({type(e).__name__}: {e})") return False - print(f"cpu emulator: ran ARM64 and read back {answer}") + _say(f"cpu emulator: ran ARM64 and read back {answer}") return answer == 42 From e0964982891d82da1a566baa68e2b464005658de Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 08:57:49 +0200 Subject: [PATCH 5/9] Report the emulator one call at a time, so a native death names the call The Windows build dies inside the emulator check with no exception - the try/except printed nothing, so there is nothing to catch and nothing in a traceback. The only evidence a hard fault leaves is the last line that got out before it. So each stage announces itself before it runs: import, construct, mem_map, mem_write, emu_start. Whichever has no answer after it is the one that killed the process. The import step also prints where unicorn was loaded from, because a bundle carrying the Python package without the native library beside it is the likeliest cause and looks identical from outside. This is instrumentation, not a fix. It buys one CI round trip and a line number. --- python/exporter/wizard.py | 62 +++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/python/exporter/wizard.py b/python/exporter/wizard.py index 1501a5a9..0e45ef51 100644 --- a/python/exporter/wizard.py +++ b/python/exporter/wizard.py @@ -1042,25 +1042,63 @@ def _emulator_runs() -> bool: and no account - and it fails a build rather than a user's machine, which is the entire point. Anisette needs this to work, because emulating Apple's ADI library is how a sign-in happens without a third-party server. + + **Reported one call at a time**, because a native fault produces no exception and no + traceback - so the only thing that says where it died is the last line that made it out. + Each stage announces itself before it runs, and the failing call is whichever one has no + answer after it. That is what turned "the Windows build dies somewhere" into a line number. """ - try: - from unicorn import UC_ARCH_ARM64, UC_MODE_ARM, Uc # noqa: PLC0415 - from unicorn.arm64_const import UC_ARM64_REG_X0 # noqa: PLC0415 - - emulator = Uc(UC_ARCH_ARM64, UC_MODE_ARM) - emulator.mem_map(0x1000, 0x1000) - emulator.mem_write(0x1000, bytes.fromhex("400580d2")) # movz x0, #42 - emulator.emu_start(0x1000, 0x1004) - answer = emulator.reg_read(UC_ARM64_REG_X0) - except Exception as e: # noqa: BLE001 - any failure here is a broken bundle, not a bug to raise - _say(f"cpu emulator: FAILED ({type(e).__name__}: {e})") - return False + state: dict = {} + + for name, step in ( + ("import", _emu_import), + ("construct", _emu_construct), + ("mem_map", _emu_map), + ("mem_write", _emu_write), + ("emu_start", _emu_run), + ): + _say(f"cpu emulator: {name} …") + try: + step(state) + except Exception as e: # noqa: BLE001 - a broken bundle, not a bug worth raising + _say(f"cpu emulator: {name} FAILED ({type(e).__name__}: {e})") + return False + answer = state["answer"] _say(f"cpu emulator: ran ARM64 and read back {answer}") return answer == 42 +def _emu_import(state: dict) -> None: + import unicorn # noqa: PLC0415 + from unicorn import arm64_const # noqa: PLC0415 + + state["unicorn"] = unicorn + state["arm64"] = arm64_const + # The path matters: a bundle can carry the Python package and miss the native library beside + # it, and then this is the last line before the process disappears. + _say(f"cpu emulator: loaded {getattr(unicorn, '__file__', '?')}") + + +def _emu_construct(state: dict) -> None: + unicorn = state["unicorn"] + state["uc"] = unicorn.Uc(unicorn.UC_ARCH_ARM64, unicorn.UC_MODE_ARM) + + +def _emu_map(state: dict) -> None: + state["uc"].mem_map(0x1000, 0x1000) + + +def _emu_write(state: dict) -> None: + state["uc"].mem_write(0x1000, bytes.fromhex("400580d2")) # movz x0, #42 + + +def _emu_run(state: dict) -> None: + state["uc"].emu_start(0x1000, 0x1004) + state["answer"] = state["uc"].reg_read(state["arm64"].UC_ARM64_REG_X0) + + if __name__ == "__main__": if "--self-test" in sys.argv[1:]: import anisette # noqa: F401, PLC0415 - imported for its path, above From ed4be642fb3cf2726ddaaa91f7fdced803f1770c Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 09:00:42 +0200 Subject: [PATCH 6/9] Ask whether the emulator runs on Windows at all, before blaming PyInstaller The frozen binary gets through import, construct, mem_map and mem_write and dies inside emu_start - so the native library loads and the bundle is not the problem. That leaves two possibilities that look the same from outside: freezing breaks execution, or executing emulated ARM64 does not work on this platform. This runs the same five calls under plain uv Python before anything is frozen. Passing means PyInstaller; failing means local Anisette cannot work on Windows at all, and the sign-in there needs a remote Anisette server - a product decision rather than a build fix. continue-on-error, because the answer is worth having either way and a red X here would bury the frozen result underneath it. --- .github/workflows/exporter-build-check.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/exporter-build-check.yml b/.github/workflows/exporter-build-check.yml index 6b4332e3..aa505b41 100644 --- a/.github/workflows/exporter-build-check.yml +++ b/.github/workflows/exporter-build-check.yml @@ -59,6 +59,23 @@ jobs: working-directory: ./python run: uv sync --frozen --no-default-groups --group build + # **Before freezing anything**, because it separates two very different faults that look + # identical from outside. The frozen binary dies inside `emu_start` after importing unicorn, + # constructing it and mapping memory - so the native library loads and the bundle is fine. + # What is not yet known is whether *executing* emulated ARM64 works on Windows at all. + # + # If this passes and the frozen one fails, it is PyInstaller. If both fail, unicorn cannot run + # here and local Anisette can never work on Windows - which is a product decision, not a build + # fix, because the sign-in would have to use a remote Anisette server instead. + # + # `continue-on-error` so the answer is recorded even when it is "no": this step is an + # experiment, and a red X here would hide the frozen result below it. + - name: Does the emulator run under plain Python on this machine + continue-on-error: true + working-directory: ./python + run: | + uv run --frozen python -c "import unicorn, sys; print('unicorn', unicorn.__file__, flush=True); uc = unicorn.Uc(unicorn.UC_ARCH_ARM64, unicorn.UC_MODE_ARM); print('constructed', flush=True); uc.mem_map(0x1000, 0x1000); uc.mem_write(0x1000, bytes.fromhex('400580d2')); print('written', flush=True); uc.emu_start(0x1000, 0x1004); from unicorn.arm64_const import UC_ARM64_REG_X0; print('x0 =', uc.reg_read(UC_ARM64_REG_X0), flush=True)" + # Kept identical to the release workflow's Windows step on purpose. If the two drift, this # stops testing what gets published, which is worse than not testing at all because it reads # as coverage. From e6b555a4e2b1059062309f29cc5bd5f2cd32b314 Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 09:05:19 +0200 Subject: [PATCH 7/9] Get the real exit code out of Windows, not bash's 127 The plain-Python run on the same machine emulates ARM64 and reads back 42, so unicorn works here and the frozen build is what breaks it. What kills the frozen one is still unnamed because bash reports any process Windows killed as 127 - which is bash's own "command not found" and carries no information. PowerShell keeps the NTSTATUS. 0xC0000005 is an access violation and 0xC0000409 is a security check, meaning Control Flow Guard or a stack cookie - and for a library that JITs its own code those point at completely different fixes. --- .github/workflows/exporter-build-check.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/exporter-build-check.yml b/.github/workflows/exporter-build-check.yml index aa505b41..70a4da17 100644 --- a/.github/workflows/exporter-build-check.yml +++ b/.github/workflows/exporter-build-check.yml @@ -102,9 +102,18 @@ jobs: # `--self-test` reaches the data files *and* runs four bytes of ARM64 through the emulator, # which is the part that is native, the part PyInstaller cannot see, and the part that killed # the process on a user's machine rather than raising anything a log could hold. + # + # PowerShell rather than bash, only so the exit code survives. bash reports a process killed + # by Windows as 127, which is its own "command not found" and says nothing; the real value is + # an NTSTATUS and it names the mechanism. 0xC0000005 is an access violation, 0xC0000409 is a + # security check - Control Flow Guard or a stack cookie - and those point at very different + # fixes for a library that JITs its own code the way unicorn does. - name: Start the binary and run its self-test working-directory: ./python/dist - shell: bash + shell: pwsh run: | ./OpenTagViewer/OpenTagViewer --version ./OpenTagViewer/OpenTagViewer --self-test + $code = $LASTEXITCODE + Write-Host ("self-test exit code: {0} (0x{1:X8})" -f $code, $code) + exit $code From c5cd7dbd6f15abb83dc788f6ad89be5a2b023873 Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 09:08:08 +0200 Subject: [PATCH 8/9] Fail the Windows check on the output, not on an exit code Windows does not set The previous commit turned this green over a binary that was still dying. A process Windows kills leaves $LASTEXITCODE unset, `exit $null` is `exit 0`, and the step reported success while the self-test never got past emu_start - the log shows the stages and then nothing, no "read back 42" and no "self-test passed". That is the worst outcome available: a check that goes green over the exact failure it was written for reads as coverage and is worse than not having one. Success is now the line the self-test prints when it finishes, and nothing else counts. The exit code is still printed, because it is evidence, but it decides nothing. --- .github/workflows/exporter-build-check.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/exporter-build-check.yml b/.github/workflows/exporter-build-check.yml index 70a4da17..f27030f4 100644 --- a/.github/workflows/exporter-build-check.yml +++ b/.github/workflows/exporter-build-check.yml @@ -111,9 +111,24 @@ jobs: - name: Start the binary and run its self-test working-directory: ./python/dist shell: pwsh + # **Asserted on the output, not on the exit code**, because the exit code cannot be trusted + # here and trusting it already produced a green run over a dead binary. A process Windows + # kills leaves `$LASTEXITCODE` unset, `exit $null` is `exit 0`, and the check passed while the + # self-test was still dying at `emu_start`. A build check that goes green over the failure it + # was written for is worse than not having one, because it reads as coverage. + # + # So success is the line the self-test prints when it finishes, and nothing else counts. run: | ./OpenTagViewer/OpenTagViewer --version - ./OpenTagViewer/OpenTagViewer --self-test + + $output = (& ./OpenTagViewer/OpenTagViewer --self-test 2>&1 | Out-String) $code = $LASTEXITCODE - Write-Host ("self-test exit code: {0} (0x{1:X8})" -f $code, $code) - exit $code + + Write-Host $output + Write-Host ("self-test exit code: '{0}'" -f $code) + + if ($output -notmatch 'self-test passed') { + Write-Host "The self-test did not report success. The last line above is the call that" + Write-Host "killed it - a native fault raises nothing, so there is no traceback to find." + exit 1 + } From 961a7e7511ff1e522da7a82a2a2862db8be1d985 Mon Sep 17 00:00:00 2001 From: "Shane B." Date: Wed, 19 Aug 2026 09:20:06 +0200 Subject: [PATCH 9/9] Turn Control Flow Guard off, which is what was killing the Windows build Named at last. The frozen binary exits 0xC0000409 from `emu_start` - STATUS_STACK_BUFFER_OVERRUN, which is what __fastfail reports for a security-check failure, and Control Flow Guard is the one in play. Every observation fits: the DLL loads, the emulator constructs, memory maps and writes, and the process dies the moment execution enters code that did not exist at link time and is therefore not in CFG's table of valid indirect-call targets. Two CI runs established it was not packaging. The same five calls under plain Python on the same Windows runner emulate ARM64 and read back 42, so unicorn works there and the bundle carries everything - it is freezing that breaks it. PyInstaller's bootloader is built with CFG on and the flag is process-wide, so no packaging option avoids it. The flag is one bit in the PE header. `scripts/clear_windows_cfg.py` clears it after the build, in both the release workflow and the check, and refuses a file whose headers it does not recognise - because a patch step that silently does nothing produces a binary that looks fixed and still dies. It gives up a real mitigation, and the script says so rather than hiding it. The alternative is an exporter that cannot sign in on Windows at all: emulating Apple's ADI library is how a login happens without handing the exchange to a third-party Anisette server. Eight tests against a synthetic PE, since the real one only exists on a Windows runner - covering the bit, the neighbouring flags it must not touch, both header widths, and the three shapes of wrong file. --- .github/workflows/exporter-build-check.yml | 17 ++- .github/workflows/macos-exporter-python.yml | 17 +++ scripts/clear_windows_cfg.py | 120 +++++++++++++++++++ scripts/test/test_clear_windows_cfg.py | 124 ++++++++++++++++++++ 4 files changed, 275 insertions(+), 3 deletions(-) create mode 100644 scripts/clear_windows_cfg.py create mode 100644 scripts/test/test_clear_windows_cfg.py diff --git a/.github/workflows/exporter-build-check.yml b/.github/workflows/exporter-build-check.yml index f27030f4..45b1d9fc 100644 --- a/.github/workflows/exporter-build-check.yml +++ b/.github/workflows/exporter-build-check.yml @@ -108,9 +108,17 @@ jobs: # an NTSTATUS and it names the mechanism. 0xC0000005 is an access violation, 0xC0000409 is a # security check - Control Flow Guard or a stack cookie - and those point at very different # fixes for a library that JITs its own code the way unicorn does. - - name: Start the binary and run its self-test - working-directory: ./python/dist - shell: pwsh + # **Control Flow Guard has to come off, or the program cannot sign in.** unicorn compiles + # ARM64 into native code at runtime and jumps to it; CFG checks every indirect call against + # targets the linker knew about, and code that did not exist at link time is not among them. + # The process is then killed by __fastfail with no exception and no traceback - observed here + # as exit `0xC0000409` from `emu_start`, and reported by a user as "parou de funcionar". + # + # PyInstaller's bootloader is built with CFG on and the flag is process-wide, so no packaging + # option avoids it. See the script for why giving up the mitigation is the right trade. + - name: Turn off Control Flow Guard, so the emulator can run + run: python scripts/clear_windows_cfg.py python/dist/OpenTagViewer/OpenTagViewer.exe + # **Asserted on the output, not on the exit code**, because the exit code cannot be trusted # here and trusting it already produced a green run over a dead binary. A process Windows # kills leaves `$LASTEXITCODE` unset, `exit $null` is `exit 0`, and the check passed while the @@ -118,6 +126,9 @@ jobs: # was written for is worse than not having one, because it reads as coverage. # # So success is the line the self-test prints when it finishes, and nothing else counts. + - name: Start the binary and run its self-test + working-directory: ./python/dist + shell: pwsh run: | ./OpenTagViewer/OpenTagViewer --version diff --git a/.github/workflows/macos-exporter-python.yml b/.github/workflows/macos-exporter-python.yml index 4c2ebb8f..cd9a14fd 100644 --- a/.github/workflows/macos-exporter-python.yml +++ b/.github/workflows/macos-exporter-python.yml @@ -262,6 +262,23 @@ jobs: --noconfirm ` ./exporter/wizard.py + # **Without this the Windows build cannot sign in**, and fails in the way that is hardest to + # report: the process vanishes, with no exception, no traceback and nothing in the log. + # + # unicorn compiles ARM64 into native code at runtime and jumps to it, which is how Apple's ADI + # library is emulated and therefore how a login happens without a third-party Anisette server. + # Control Flow Guard checks every indirect call against targets the linker knew about, and code + # that did not exist at link time is not among them, so the jump is refused and __fastfail + # terminates the process. PyInstaller's bootloader is built with CFG on and the flag applies to + # the whole process, so nothing about how the Python side is packaged avoids it. + # + # Reported by a user as "OpenTagViewer.exe parou de funcionar" on 1.2.0, and reproduced in + # exporter-build-check.yml as exit 0xC0000409 from `emu_start`. The script explains why giving + # up the mitigation is the right trade for this program. + - name: Turn off Control Flow Guard (Windows), so the emulator can run + if: matrix.os == 'windows' + run: python scripts/clear_windows_cfg.py python/dist/OpenTagViewer/OpenTagViewer.exe + # No --windowed on Linux: it means "no console window" on the platforms that have that # concept, and there is no bundle format here for it to produce either way. # --collect-all unicorn is not optional, and its absence is invisible until somebody runs the diff --git a/scripts/clear_windows_cfg.py b/scripts/clear_windows_cfg.py new file mode 100644 index 00000000..8eaa6545 --- /dev/null +++ b/scripts/clear_windows_cfg.py @@ -0,0 +1,120 @@ +""" +Turn off Control Flow Guard on a built Windows executable, so a JIT can run inside it. + +**Why this exists.** The exporter signs in by emulating Apple's ADI library, which `unicorn` does +by compiling ARM64 into native code at runtime and jumping to it. Control Flow Guard is a Windows +mitigation that checks every indirect call against a table of targets the linker knew about, and +code that did not exist at link time is not in that table. So the jump is rejected and the process +is terminated by `__fastfail` - **no exception, no traceback, nothing in any log**, because the +whole point of the mechanism is to stop the process before anything can handle it. + +That is exactly what a user reported as `OpenTagViewer.exe parou de funcionar`, and what CI +reproduced: `import`, construct, `mem_map` and `mem_write` all succeed, and `emu_start` returns +exit code `0xC0000409`. + +PyInstaller's bootloader is built with CFG enabled and the flag applies to the whole process, so +nothing about how the Python side is packaged can avoid it. The flag lives in one bit of the PE +header, and clearing it after the build is the smallest change that makes the program work. + +**This is a real trade, not a formality.** CFG is a genuine hardening measure and this gives it up +for the whole executable. It is given up because the alternative is an exporter that cannot sign in +on Windows at all: the emulator is not an optional component, it is how the login works without +handing the exchange to a third-party Anisette server. Every program that JITs - browsers, .NET, +every language runtime with a tracing compiler - either disables CFG or registers its targets with +`SetProcessValidCallTargets`, and the second is not reachable from here. + +Usage: + + python scripts/clear_windows_cfg.py path/to/OpenTagViewer.exe +""" + +from __future__ import annotations + +import struct +import sys +from pathlib import Path + +# IMAGE_DLLCHARACTERISTICS_GUARD_CF. One bit, and the difference between a program that runs and +# one that is killed on its first jump into compiled code. +GUARD_CF = 0x4000 + +PE32 = 0x10B +PE32_PLUS = 0x20B + +# Offset of DllCharacteristics within the optional header. The same for PE32 and PE32+: the two +# differ earlier - PE32+ drops BaseOfData and widens ImageBase to eight bytes - and the layouts +# realign at SectionAlignment, well before this field. +DLL_CHARACTERISTICS_OFFSET = 0x46 + + +class NotAPortableExecutable(Exception): + """The file is not a PE image, so there is no flag here to change.""" + + +def clear_guard_cf(path: Path) -> bool: + """ + Clear the CFG bit in a PE image, in place. + + :returns: Whether the bit was set to begin with, so a caller can tell "turned it off" from + "there was nothing to turn off" - which are different, and only the first is expected. + :raises NotAPortableExecutable: If the file does not have the headers this needs. Raised + rather than ignored: silently doing nothing to the wrong file would produce a build that + looks patched and still dies. + """ + data = bytearray(path.read_bytes()) + + if data[:2] != b"MZ": + msg = f"{path} does not start with MZ, so it is not a PE image." + raise NotAPortableExecutable(msg) + + pe_offset = struct.unpack_from(" int: + if len(argv) != 1: + print(__doc__) + return 2 + + path = Path(argv[0]) + + try: + cleared = clear_guard_cf(path) + except (OSError, NotAPortableExecutable) as e: + print(f"Could not clear Control Flow Guard on {path}: {e}", file=sys.stderr) + return 1 + + if cleared: + print(f"Control Flow Guard turned off on {path}") + else: + # Worth saying rather than passing quietly. It means either the toolchain stopped setting + # it - in which case this script has become unnecessary - or the wrong file was patched, + # and those want very different follow-ups. + print(f"Control Flow Guard was already off on {path}; nothing changed") + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test/test_clear_windows_cfg.py b/scripts/test/test_clear_windows_cfg.py new file mode 100644 index 00000000..9c1806b5 --- /dev/null +++ b/scripts/test/test_clear_windows_cfg.py @@ -0,0 +1,124 @@ +""" +Clearing the Control Flow Guard bit from a built Windows executable. + +**Tested on a synthetic PE, because the real one only exists on a Windows runner.** The header +layout is fixed by the PE specification, so a handful of bytes in the right places exercises the +same arithmetic the real file does - and the alternative is a script whose only test is a CI job +on another operating system, which is how a patch that silently does nothing gets shipped. + +What matters here is not only that the bit is cleared. It is that the wrong file is *refused*: +a build step that quietly succeeds on something it did not understand produces a binary that looks +patched and still dies on a user's machine, with the one symptom that leaves no traceback. +""" + +from __future__ import annotations + +import struct +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from clear_windows_cfg import ( # noqa: E402 + GUARD_CF, + PE32, + PE32_PLUS, + NotAPortableExecutable, + clear_guard_cf, +) + +PE_OFFSET = 0x80 +OPTIONAL = PE_OFFSET + 4 + 20 +FIELD = OPTIONAL + 0x46 + + +def build_pe(characteristics: int, *, magic: int = PE32_PLUS) -> bytes: + """The smallest thing that is shaped like a PE image as far as this script is concerned.""" + data = bytearray(FIELD + 2) + data[0:2] = b"MZ" + struct.pack_into(" int: + return struct.unpack_from(" Path: + path = tmp_path / "OpenTagViewer.exe" + path.write_bytes(build_pe(characteristics, **kwargs)) + return path + + return write + + +class TestClearingTheBit: + def test_it_turns_control_flow_guard_off(self, executable): + path = executable(GUARD_CF) + + assert clear_guard_cf(path) is True + assert characteristics_of(path) & GUARD_CF == 0 + + def test_it_leaves_every_other_flag_alone(self, executable): + # The field carries ASLR, DEP and the rest beside it. Clearing the wrong bits would turn a + # mitigation problem into a different mitigation problem, on a binary handed to users. + others = 0x0040 | 0x0100 | 0x8000 # DYNAMIC_BASE, NX_COMPAT, TERMINAL_SERVER_AWARE + path = executable(GUARD_CF | others) + + clear_guard_cf(path) + + assert characteristics_of(path) == others + + def test_a_file_that_never_had_it_reports_no_change(self, executable): + # Distinct from success, and the caller says so: it means either the toolchain stopped + # setting the bit, or the wrong file was patched. Those want different follow-ups. + path = executable(0x0100) + + assert clear_guard_cf(path) is False + assert characteristics_of(path) == 0x0100 + + @pytest.mark.parametrize("magic", [PE32, PE32_PLUS]) + def test_both_header_widths_are_handled(self, executable, magic): + # PE32 and PE32+ differ earlier in the optional header and realign well before this field. + path = executable(GUARD_CF, magic=magic) + + assert clear_guard_cf(path) is True + + +class TestRefusingTheWrongFile: + """ + Each of these would otherwise be a build step that appears to work. + + That is the failure worth guarding: the exe reaches a user unpatched, and the symptom is a + process that vanishes without an exception, which is the hardest thing there is to report. + """ + + def test_something_that_is_not_an_executable(self, tmp_path): + path = tmp_path / "notes.txt" + path.write_bytes(b"this is not a program") + + with pytest.raises(NotAPortableExecutable, match="does not start with MZ"): + clear_guard_cf(path) + + def test_a_dos_header_pointing_at_nothing(self, tmp_path): + data = bytearray(build_pe(GUARD_CF)) + data[PE_OFFSET:PE_OFFSET + 4] = b"junk" + path = tmp_path / "OpenTagViewer.exe" + path.write_bytes(bytes(data)) + + with pytest.raises(NotAPortableExecutable, match="no PE signature"): + clear_guard_cf(path) + + def test_an_optional_header_of_an_unknown_shape(self, executable): + path = executable(GUARD_CF, magic=0x1234) + + with pytest.raises(NotAPortableExecutable, match="neither PE32 nor PE32"): + clear_guard_cf(path)