Merge branch 'Blosc:main' into matmul

Author: ricardosp4

.github/workflows/cibuildwheels.yml

@@ -11,8 +11,9 @@ on:
 
 env:
   CIBW_BUILD_VERBOSITY: 1
-  # Skip aarch64 for now, as it is emulated on GitHub Actions and takes too long
-  CIBW_TEST_SKIP: "*linux*aarch64*"
+  # Skip testing on aarch64 for now, as it is emulated on GitHub Actions and takes too long
+  # Now that github provides native arm64 runners, we can enable tests again
+  # CIBW_TEST_SKIP: "*linux*aarch64*"
 
 jobs:
 
@@ -27,11 +28,11 @@ jobs:
       CIBW_ARCHS_MACOS: "x86_64 arm64"
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        # arch: [x86_64, aarch64]
+        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest]
+        arch: [x86_64, aarch64]
         # aarch64 is emulated, and it is giving issues with cibuildwheel
         # Just build for x86_64 for now (Mac arm64 is already covered by cibuildwheel)
-        arch: [x86_64]
+        # arch: [x86_64]
         cibw_build: ["cp3{11,12,13}-*"]
         p_ver: ["3.11-3.13"]
         exclude:
@@ -40,6 +41,11 @@ jobs:
           # cibuild is already in charge to build aarch64 (see CIBW_ARCHS_MACOS)
           - os: macos-latest
             arch: aarch64
+          # Mutually exclude aarch64 and x86_64 for ubuntu-24.04-arm and ubuntu-latest
+          - os: ubuntu-24.04-arm
+            arch: x86_64
+          - os: ubuntu-latest
+            arch: aarch64
 
     steps:
       - name: Checkout repo
@@ -51,6 +57,7 @@ jobs:
           # Use the most recent released python
           python-version: '3.x'
 
+      # For some reason, this is still needed, even when using new arm64 runners
       - name: Set up QEMU
         if: ${{ matrix.arch == 'aarch64' }}
         uses: docker/setup-qemu-action@v3
@@ -73,7 +80,6 @@ jobs:
           name: wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.p_ver }}
           path: ./wheelhouse/*.whl
 
-
   build_sdist:
     name: Build sdist
     runs-on: ubuntu-latest
@@ -96,25 +102,25 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Build sdist
-        run: pipx run build --sdist
+        run: pipx run build --sdist --outdir ./wheelhouse
 
       - name: Upload sdist package
         uses: actions/upload-artifact@v4
         with:
           name: wheels-source
-          path: dist/*.tar.gz
+          path: wheelhouse/*.tar.gz
 
       - name: Build building extension from sdist package
         run: |
-          cd ./dist
+          cd ./wheelhouse
           tar -xzf blosc2-*.tar.gz
           cd ./blosc2-*/
           pip install pip --upgrade
           pip install --break-system-packages -e .[test]
 
       - name: Test sdist package with pytest
         run: |
-          cd ./dist/blosc2-*/
+          cd ./wheelhouse/blosc2-*/
           pytest
 
   upload_pypi:
@@ -125,7 +131,8 @@ jobs:
     steps:
       - uses: actions/download-artifact@v4
         with:
-          pattern: wheels-*   # Download the wheels artifact
+          name: wheels-*
+          path: ./wheelhouse
           merge-multiple: true  # Merge all the wheels artifacts into one directory
 
       - uses: pypa/gh-action-pypi-publish@release/v1

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.4
     hooks:
       - id: ruff
         args: ["--fix", "--show-fixes"]

ANNOUNCE.rst

@@ -1,9 +1,21 @@
-Announcing Python-Blosc2 3.0.0 final
-====================================
+Announcing Python-Blosc2 3.1.0 (and 3.1.1)
+==========================================
 
-The Blosc development team is pleased to announce the final release for
-Python-Blosc2 3.0.0. Now, we will be producing conda(-forge) packages,
-as well as providing wheels for the most common platforms, as usual.
+This is a minor release where we optimized the performance of the
+internal compute engine, as well as indexing for NDArrays.  We also
+added new API:
+
+* blosc2.evaluate: a drop-in replacement of numexpr.evaluate().  This
+  allows to evaluate expressions with combinations of NDArrays and NumPy
+  arrays.  In addition, it has the next improvements:
+
+  - More functionality than numexpr (e.g. reductions).
+  - Follow casting rules of NumPy more closely.
+  - Use both NumPy arrays and Blosc2 NDArrays in the same expression.
+
+* blosc2.jit: a decorator to compile a function with a NumPy expression
+  using the compute engine of Blosc2.  Useful to speed up the evaluation
+  of pure NumPy expressions at runtime.
 
 You can think of Python-Blosc2 3.0 as an extension of NumPy/numexpr that:
 
@@ -17,7 +29,7 @@ You can think of Python-Blosc2 3.0 as an extension of NumPy/numexpr that:
 
 Install it with::
 
-    pip install blosc2==3.0.0   # if you prefer wheels
+    pip install blosc2 --update   # if you prefer wheels
     conda install -c conda-forge python-blosc2 mkl  # if you prefer conda and MKL
 
 For more info, you can have a look at the release notes in:

CMakeLists.txt

@@ -50,7 +50,7 @@ else()
     include(FetchContent)
     FetchContent_Declare(blosc2
         GIT_REPOSITORY https://github.com/Blosc/c-blosc2
-        GIT_TAG d92b88e535f168fae45874b936b187c790fc9d59
+        GIT_TAG 34db770e436aa4ceaa9b9110948f8dac1f1f443d
     )
     FetchContent_MakeAvailable(blosc2)
     include_directories("${blosc2_SOURCE_DIR}/include")

RELEASE_NOTES.md

@@ -1,10 +1,66 @@
 # Release notes
 
-## Changes from 3.0.0 to 3.0.1
+## Changes from 3.1.1 to 3.1.2
 
-XXX version-specific blurb XXX
+* Structured arrays can be larger than 255 bytes now.  This was a limitation
+  in the previous versions, but now it is gone (the new limit is ~512 MB,
+  which I hope will be enough for some time).
 
 
+## Changes from 3.1.0 to 3.1.1
+
+* Quick release to fix an issue with version number in the package (was reporting 3.0.0
+  instead of 3.1.0).
+
+
+## Changes from 3.0.0 to 3.1.0
+
+### Improvements
+
+* Optimizations for the compute engine. Now, it is faster and uses less memory.
+  In particular, careful attention has been paid to the memory handling, as
+  this is the main bottleneck for the compute engine in many instances.
+
+* Improved detection of CPU cache sizes for Linux and macOS.  In particular,
+  support for multi-CCX (AMD EPYC) and multi-socket systems has been implemented.
+  Now, the library should be able to detect the cache sizes for most of the
+  CPUs out there (specially on Linux).
+
+* Optimization on NDArray slicing when the slice is a single chunk.  This is a
+  common operation when working with NDArray instances, and now it is faster.
+
+### New API functions and decorators
+
+* New `blosc2.evaluate()` function for evaluating expressions on NDArray/NumPy
+  instances.  This a drop-in replacement of `numexpr.evaluate()`, but with the
+  next improvements:
+  - More functionality than numexpr (e.g. reductions).
+  - Follow casting rules of NumPy more closely.
+  - Use both NumPy arrays and Blosc2 NDArrays in the same expression.
+
+  See [here](https://www.blosc.org/python-blosc2/reference/autofiles/utilities/blosc2.evaluate.html)
+  for more information.
+
+* New `blosc2.jit` decorator for allowing NumPy expressions to be computed
+  using the Blosc2 compute engine.  This is a powerful feature that allows for
+  efficient computations on compressed data, and supports advanced features like
+  reductions, filters and broadcasting.  See
+  [here](https://www.blosc.org/python-blosc2/reference/autofiles/utilities/blosc2.jit.html)
+  for more information.
+
+* Support `out=` in `blosc2.mean()`, `blosc2.std()` and `blosc2.var()` reductions
+  (besides `blosc2.sum()` and `blosc2.prod()`).
+
+
+### Others
+
+* Bumped to use latest C-Blosc2 sources (2.16.0).
+
+* The cache for cpuinfo is now stored in `${HOME}/.cache/python-blosc2/cpuinfo.json`
+  instead of `${HOME}/.blosc2-cpuinfo.json`; you can get rid of the latter, as
+  the former is more standard (see PR #360). Thanks to Jonas Lundholm Bertelsen
+  (@jonaslb).
+
 ## Changes from 3.0.0-rc.3 to 3.0.0
 
 * A persistent cache for cpuinfo (stored in `$HOME/.blosc2-cpuinfo.json`) is

RELEASING.rst

@@ -4,7 +4,13 @@ Python-Blosc2 release procedure
 Preliminaries
 -------------
 
-* Set the version number in ``pyproject.toml`` to the new version number (e.g. ``X.Y.Z``).
+* Set the version number for the release by using::
+
+    python update_version.py X.Y.Z
+
+  and double-check the updated version number in ``pyproject.toml`` and with::
+
+    python -c "import blosc2; print(blosc2.__version__)"
 
 * Make sure that the c-blosc2 repository is updated to the latest version (or a specific
   version that will be documented in the ``RELEASE_NOTES.md``). In ``CMakeLists.txt`` edit::
@@ -18,11 +24,10 @@ Preliminaries
 
 * Make sure that the current main branch is passing the tests in continuous integration.
 
-* Build the package and make sure that::
+* Build the package and make sure that tests are passing::
 
-    python -c "import blosc2; blosc2.print_versions()"
-
-  is printing the correct versions.
+    pip install -e ".[test]"
+    pytest
 
 * Make sure that ``RELEASE_NOTES.md`` and ``ANNOUNCE.rst`` are up to date with the
   latest news in the release.
@@ -53,19 +58,35 @@ Tagging
 
     git push --tags
 
+* If you happen to have to delete the tag, such as artifacts demonstrates a fault, first delete it locally::
+
+    git tag --delete vX.Y.Z
+
+  and then remotely on Github:
+
+    git push --delete origin vX.Y.Z
+
 * Make sure that the tag is passing the tests in continuous integration (this
-  may take more than an hour).
+  may take about 30 min).
+
+* In case the automatic upload to PyPI fails, you can upload the package
+  wheels (and tarball!) by downloading the artifacts manually, copying to
+  an empty dir (say dist), and upload to PyPI with::
+
+    rm dist/*
+    # download artifacts from the tag in github
+    twine upload --repository blosc dist/*
 
 * Update the latest release in the ``doc/python-blosc2.rst`` file with the new version
   number (and date?).  Do a commit::
 
     git commit -a -m "Update latest release in doc"
     git push
 
-* Go to ``Blosc/blogsite`` repo, then to "Actions", click on the most recent
-  workflow run (at the top of the list), and then click on the "Re-run all
-  jobs" button to regenerate the documentation and check that it has been
-  correctly updated in https://www.blosc.org.
+* Go to ``https://github.com/Blosc/blogsite`` repo, then to "Actions", click
+  on the most recent workflow run (at the top of the list), and then click on
+  the "Re-run all jobs" button to regenerate the documentation and check that
+  it has been correctly updated in https://www.blosc.org.
 
 
 Checking packaging
@@ -93,6 +114,7 @@ Announcing
   skeleton (or possibly as the definitive version). Start the subject with ANN:.
 
 * Announce in Mastodon via https://fosstodon.org/@Blosc2 account and rejoice.
+  Announce it in Bluesky too.
 
 
 Post-release actions
@@ -109,8 +131,9 @@ Post-release actions
 
     XXX version-specific blurb XXX
 
-* Update the version number in ``pyproject.toml`` to the next version number
-  (e.g. ``X.Y.(Z+1).dev``).
+* Update the version number in ``pyproject.toml`` and ``version.py`` to the next version number::
+
+    python update_version.py X.Y.(Z+1).dev0
 
 * Commit your changes with::