Merge pull request #367 from Blosc/pyodide

Support for building WASM32 wheels.

There is a need to get rid of `httpx` in favor of `requests`, but that can come later.

Author: FrancescAlted

.github/workflows/wasm.yml

@@ -0,0 +1,63 @@
+name: Python wheels for WASM
+on:
+  push:
+    tags:
+      - '*'
+  pull_request:
+    branches:
+      - main
+
+env:
+  CIBW_BUILD_VERBOSITY: 1
+  # cibuildwheel cannot choose for a specified version of pyodide yet
+  # PYODIDE_VERSION: 0.27.2
+
+jobs:
+  build_wheels_wasm:
+    name: Build and test wheels for WASM on ${{ matrix.os }} for ${{ matrix.p_ver }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      CIBW_BUILD: ${{ matrix.cibw_build }}
+      CMAKE_ARGS: "-DWITH_OPTIM=OFF"
+      CIBW_TEST_COMMAND: "pytest {project}/tests/ndarray/test_reductions.py"
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        cibw_build: ["cp3{11,12,13}-*"]
+        p_ver: ["3.11-3.13"]
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake
+
+      - name: Install cibuildwheel
+        run: pip install cibuildwheel
+
+      - name: Build wheels
+        # Testing is automaticall made by cibuildwheel
+        run: cibuildwheel --platform pyodide
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-wasm-${{ matrix.os }}-${{ matrix.p_ver }}
+          path: ./wheelhouse/*.whl
+
+      - name: Upload wheel to release
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload ${GITHUB_REF_NAME} ./wheelhouse/*.whl

pyproject.toml

@@ -31,13 +31,14 @@ classifiers = [
 requires-python = ">=3.11"
 # Follow guidelines from https://scientific-python.org/specs/spec-0000/
 dependencies = [
-    "numpy>=1.25.0",
+    "numpy>=1.26",
+    #"numpy>=2",
     "ndindex",
     "msgpack",
-    "numexpr",
-    "py-cpuinfo",
-    "httpx",
     "platformdirs",
+    "numexpr; platform_machine != 'wasm32'",
+    "py-cpuinfo; platform_machine != 'wasm32'",
+    "httpx; platform_machine != 'wasm32'",
 ]
 version = "3.1.2.dev0"
 
@@ -59,8 +60,8 @@ dev = [
 ]
 test = [
     "pytest",
-    "psutil",
-    "torch",
+    "psutil; platform_machine != 'wasm32'",
+    "torch; platform_machine != 'wasm32'",
 ]
 doc = [
     "sphinx>=8",
@@ -81,8 +82,11 @@ build-verbosity = 1
 # Skip unsupported python versions as well as 32-bit platforms, which are not supported anymore.
 skip = "*-manylinux_i686 cp*-win32 *_ppc64le *_s390x *musllinux*"
 # We won't require torch when testing wheels to avoid building/running torch on slow platforms
-test-requires = "pytest psutil"
-test-command = "pytest {project}/tests"
+#test-requires = "pytest psutil"
+test-requires = "pytest"
+#test-command = "pytest {project}/tests"  # default command
+# Use a simpler command here, and let the workflow .yml file to set the command
+test-command = "python -c \"import blosc2; blosc2.print_versions()\""
 # Manylinux 2014 will be the default for x86_64 and aarch64
 manylinux-x86_64-image = "manylinux2014"
 manylinux-aarch64-image = "manylinux2014"

src/blosc2/__init__.py

@@ -10,9 +10,18 @@
 # ruff: noqa: E402 - Module level import not at top of file
 # ruff: noqa: F401 - `var` imported but unused
 
+import platform
 from enum import Enum
 
-import numexpr
+# Do the platform check once at module level
+IS_WASM = platform.machine() == "wasm32"
+# IS_WASM = True  # for testing (comment this line out for production)
+"""
+Flag for WebAssembly platform.
+"""
+
+if not IS_WASM:
+    import numexpr
 
 from .version import __version__
 
@@ -200,7 +209,9 @@ class Tuner(Enum):
 # Experiments say that, when using a large number of threads, it is better to not use them all
 if nthreads > 16:
     nthreads -= nthreads // 8
-numexpr.set_num_threads(nthreads)
+if not IS_WASM:
+    # WASM does not support threading
+    numexpr.set_num_threads(nthreads)
 
 # This import must be before ndarray and schunk
 from .storage import (  # noqa: I001

src/blosc2/c2array.py

@@ -15,11 +15,13 @@
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
-import httpx
 import numpy as np
 
 import blosc2
 
+if not blosc2.IS_WASM:
+    import httpx
+
 _subscriber_data = {
     "urlbase": os.environ.get("BLOSC_C2URLBASE"),
     "auth_token": "",

src/blosc2/core.py

@@ -24,13 +24,15 @@
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any
 
-import cpuinfo
 import numpy as np
 import platformdirs
 
 import blosc2
 from blosc2 import blosc2_ext
 
+if not blosc2.IS_WASM:
+    import cpuinfo
+
 if TYPE_CHECKING:
     from collections.abc import Callable
 
@@ -1119,13 +1121,19 @@ def print_versions():
     for clib in sorted(clib_versions.keys()):
         print(f"  {clib}: {clib_versions[clib]}")
     print(f"NumPy version: {np.__version__}")
+    if not blosc2.IS_WASM:
+        import numexpr
+
+        print(f"numexpr version: {numexpr.__version__}")
     print(f"Python version: {sys.version}")
     (sysname, _nodename, release, version, machine, processor) = platform.uname()
     print(f"Platform: {sysname}-{release}-{machine} ({version})")
     if sysname == "Linux":
         distro = os_release_pretty_name()
         if distro:
             print(f"Linux dist: {distro}")
+    if blosc2.IS_WASM:
+        processor = "wasm32"
     if not processor:
         processor = "not recognized"
     print(f"Processor: {processor}")
@@ -1202,6 +1210,17 @@ def linux_cache_size(cache_level: int, default_size: int) -> int:
 
 
 def _get_cpu_info():
+    if blosc2.IS_WASM:
+        # Emscripten/wasm32 does not have access to CPU information.
+        # Populate it with some reasonable defaults.
+        return {
+            "brand": "Emscripten",
+            "arch": "wasm32",
+            "count": 1,
+            "l1_data_cache_size": 32 * 1024,
+            "l2_cache_size": 256 * 1024,
+            "l3_cache_size": 1024 * 1024,
+        }
     cpu_info = cpuinfo.get_cpu_info()
     # cpuinfo does not correctly retrieve the cache sizes for Apple Silicon, so do it manually
     if platform.system() == "Darwin":