Windows runtime library_path alternative

Windows has no runtime_library path that's practical for us
  - require user to set PATH specificaly before running python
  - require user to use os.add_dll_directory() before `import python`
  - require user to load libzim DLL (via `ctypes.CDLL()`) in memory before `import python`
  - require us to change our API to use of of those trick before importing wrapper

Most practical solution found was thus to install our companion DLLs (libzim and libicu)
next to the wrapper one.
This is done via a new command that edits the built wheel

Added conditional support for _DEBUG as this is required to link against debug libzim DLLs

Author: rgaudin

MANIFEST.in

@@ -15,3 +15,4 @@ exclude *.egg-info/*
 exclude libzim/*.dylib
 exclude libzim/*.so
 exclude libzim/*.so.*
+exclude libzim/*.dll

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = [ "setuptools == 68.2.2", "wheel == 0.41.3", "cython == 3.0.5" ]
+requires = [ "setuptools == 68.2.2", "wheel == 0.41.3", "cython == 3.0.5", "delocate==0.12.0" ]
 build-backend = "setuptools.build_meta"
 
 [tool.black]
@@ -24,6 +24,9 @@ manylinux-aarch64-image = "manylinux_2_28"
 manylinux-pypy_x86_64-image = "manylinux_2_28"
 manylinux-pypy_aarch64-image = "manylinux_2_28"
 
+[tool.cibuildwheel.windows]
+repair-wheel-command = "python.exe setup.py repair_win_wheel --destdir={dest_dir} --wheel={wheel}"
+
 [tool.cibuildwheel.linux]
 archs = ["x86_64", "aarch64"]
 

setup.cfg

@@ -50,6 +50,12 @@ test_requires =
 libzim =
     libzim.9.dylib
     libzim.so.9
+    zim-9.dll
+    icuuc74.dll
+    icutu74.dll
+    icuio74.dll
+    icuin74.dll
+    icudt74.dll
 
 [isort]
 profile = black

setup.py

@@ -9,6 +9,8 @@
 
 The Cython and Cythonize compilation is done automatically by the build backend"""
 
+from __future__ import annotations
+
 import os
 import pathlib
 import platform as sysplatform
@@ -21,10 +23,10 @@
 import urllib.request
 from ctypes.util import find_library
 from pathlib import Path
-from typing import Optional, Tuple
 
 from Cython.Build import cythonize
 from Cython.Distutils.build_ext import new_build_ext as build_ext
+from delocate.wheeltools import InWheel
 from setuptools import Command, Extension, setup
 
 
@@ -42,7 +44,10 @@ class Config:
     apple_signing_keychain: str = os.getenv("APPLE_SIGNING_KEYCHAIN_PATH")
     apple_signing_keychain_profile: str = os.getenv("APPLE_SIGNING_KEYCHAIN_PROFILE")
 
-    supported_platforms = {
+    # windows
+    _msvc_debug: bool = bool(os.getenv("MSVC_DEBUG"))
+
+    supported_platforms = {  # noqa: RUF012
         "Darwin": ["x86_64", "arm64"],
         "Linux": ["x86_64", "aarch64"],
         "Linux-musl": ["x86_64", "aarch64"],
@@ -52,8 +57,9 @@ class Config:
     base_dir: pathlib.Path = Path(__file__).parent
 
     # Avoid running cythonize on `setup.py clean` and similar
-    buildless_commands: Tuple[str] = (
+    buildless_commands: tuple[str, ...] = (
         "clean",
+        "repair_win_wheel",
         "--help",
         "egg_info",
         "--version",
@@ -153,7 +159,15 @@ def wants_universal(self) -> bool:
             "universal2"
         )
 
-    def get_download_filename(self, arch: Optional[str] = None) -> str:
+    @property
+    def use_msvc_debug(self) -> bool:
+        """whether to add _DEBUG define to compilation
+
+        requires having python debug binaries installed.
+        mandatory for compiling against libzim nighlies"""
+        return self._msvc_debug or self.is_nightly
+
+    def get_download_filename(self, arch: str | None = None) -> str:
         """filename to download to get binary libzim for platform/arch"""
         arch = arch or self.arch
 
@@ -276,11 +290,11 @@ def _install_from(self, folder: pathlib.Path):
             print(f"{fpath} -> {libzim_dir / fpath.name}")
             os.replace(fpath, libzim_dir / fpath.name)
         # windows has different folder and name
-        for fpath in folder.joinpath("bin").rglob("zim-*.dll"):
-            print(f"{fpath} -> {libzim_dir / fpath.name}")
-            os.replace(fpath, libzim_dir / fpath.name)
-        # windows again, not sure its required at all
-        for fpath in folder.joinpath("lib").rglob("zim.lib"):
+        for fpath in (
+            list(folder.joinpath("bin").rglob("zim-*.dll"))
+            + list(folder.joinpath("bin").rglob("icu*.dll"))
+            + list(folder.joinpath("lib").rglob("zim.lib"))
+        ):
             print(f"{fpath} -> {libzim_dir / fpath.name}")
             os.replace(fpath, libzim_dir / fpath.name)
 
@@ -320,6 +334,18 @@ def cleanup(self):
                 print("removing downloaded headers")
                 shutil.rmtree(self.header_file.parent, ignore_errors=True)
 
+    def repair_windows_wheel(self, wheel: Path, dest_dir: Path):
+        """opens windows wheels in target folder and moves all DLLs files inside
+        subdirectories of the wheel to the root one (where wrapper is expected)"""
+
+        dest_wheel = dest_dir / wheel.name
+        with InWheel(str(wheel), str(dest_wheel)) as wheel_dir_path:
+            print(f"repairing {wheel.name} for Windows (DLLs next to wrapper)")
+            wheel_dir = Path(wheel_dir_path)
+            for dll in wheel_dir.joinpath("libzim").rglob("*.dll"):
+                print(f"> moving {dll} using {dll.relative_to(wheel_dir).parent}")
+                dll.replace(wheel_dir / dll.name)
+
     @property
     def header_file(self) -> pathlib.Path:
         return self.base_dir / "include" / "zim" / "zim.h"
@@ -381,14 +407,19 @@ def get_cython_extension():
         print("Using local libzim binary. Set `USE_SYSTEM_LIBZIM` otherwise.")
         include_dirs.append("include")
         library_dirs = ["libzim"]
-        runtime_library_dirs = (
-            [f"@loader_path/libzim/{config.libzim_fname}"]
-            if sysplatform == "Darwin"
-            else ["$ORIGIN/libzim/"]
-        )
+
+        if config.platform != "Windows":
+            runtime_library_dirs = (
+                [f"@loader_path/libzim/{config.libzim_fname}"]
+                if sysplatform == "Darwin"
+                else ["$ORIGIN/libzim/"]
+            )
 
     extra_compile_args = ["-std=c++11", "-Wall"]
-    if config.platform != "Windows":
+    if config.platform == "Windows":
+        extra_compile_args.append("/MDd" if config.use_msvc_debug else "/MD")
+        ...
+    else:
         extra_compile_args.append("-Wextra")
 
     wrapper_extension = Extension(
@@ -542,7 +573,29 @@ def run(self):
         config.cleanup()
 
 
-if len(sys.argv) == 2 and sys.argv[1] in config.buildless_commands:
+class RepairWindowsWheel(Command):
+    user_options = [  # noqa: RUF012
+        ("wheel=", None, "Wheel to repair"),
+        ("destdir=", None, "Destination folder for repaired wheels"),
+    ]
+
+    def initialize_options(self):
+        self.wheel = None
+        self.destdir = None
+
+    def finalize_options(self):
+        assert Path(self.wheel).exists(), "wheel file does not exists"
+        assert (
+            Path(self.destdir).exists() and Path(self.destdir).is_dir()
+        ), "dest_dir does not exists"
+
+    def run(self):
+        config.repair_windows_wheel(wheel=Path(self.wheel), dest_dir=Path(self.destdir))
+
+
+if len(sys.argv) == 1 or (
+    len(sys.argv) == 2 and sys.argv[1] in config.buildless_commands
+):
     ext_modules = None
 else:
     ext_modules = get_cython_extension()
@@ -552,6 +605,7 @@ def run(self):
         "build_ext": LibzimBuildExt,
         "download_libzim": DownloadLibzim,
         "clean": LibzimClean,
+        "repair_win_wheel": RepairWindowsWheel,
     },
     ext_modules=ext_modules,
 )

tests/test_libzim_creator.py

@@ -168,7 +168,7 @@ def get_creator_output(fpath, verbose):
     return ps.stdout
 
 
-@pytest.mark.parametrize("verbose", [(True, False)])
+@pytest.mark.parametrize("verbose", [True, False])
 def test_creator_verbose(fpath, verbose):
     output = get_creator_output(fpath, verbose).strip()
     lines = output.splitlines()