Merge branch 'main' into addStack

Luke Shaw · Luke Shaw · commit 5c933ecede96 · 2025-06-20T13:01:21.000+02:00
diff --git a/.github/workflows/cibuildwheels.yml b/.github/workflows/cibuildwheels.yml
@@ -83,7 +83,7 @@ jobs:
           arch: amd64
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.23
+        uses: pypa/cibuildwheel@v3.0
 
       - name: Make sdist
         if: ${{ matrix.os == 'ubuntu-latest' }}
diff --git a/RELEASING.rst b/RELEASING.rst
@@ -110,7 +110,8 @@ Checking packaging
 
     [upload e.g.:] blosc2-3.2.0-cp312-cp312-pyodide_2024_0_wasm32.whl
 
-
+The wheels may be downloaded by going to "Actions->Python wheels for WASM" and selecting the completed workflow run for the version you are releasing.
+  Then, go to the "Artifacts" dropdown and download the WASM wheel file.
 Announcing
 ----------
 
diff --git a/bench/ndarray/concatenate.py b/bench/ndarray/concatenate.py
@@ -14,7 +14,8 @@
 from matplotlib.ticker import ScalarFormatter
 
 
-def run_benchmark(num_arrays=10, size=500, aligned_chunks=False, axis=0, codec=blosc2.Codec.ZSTD):
+def run_benchmark(num_arrays=10, size=500, aligned_chunks=False, axis=0,
+                  dtype=np.float64, datadist="linspace", codec=blosc2.Codec.ZSTD):
     """
     Benchmark blosc2.concatenate performance with different chunk alignments.
 
@@ -23,6 +24,9 @@ def run_benchmark(num_arrays=10, size=500, aligned_chunks=False, axis=0, codec=b
     - size: Base size for array dimensions
     - aligned_chunks: Whether to use aligned chunk shapes
     - axis: Axis along which to concatenate (0 or 1)
+    - dtype: Data type for the arrays (default is np.float64)
+    - datadist: Distribution of data in arrays (default is "linspace")
+    - codec: Codec to use for compression (default is blosc2.Codec.ZSTD)
 
     Returns:
     - duration: Time taken in seconds
@@ -39,20 +43,28 @@ def run_benchmark(num_arrays=10, size=500, aligned_chunks=False, axis=0, codec=b
         raise ValueError("Only axis 0 and 1 are supported")
 
     # Create appropriate chunk shapes
+    chunks, blocks = blosc2.compute_chunks_blocks(shapes[0], dtype=dtype, cparams=blosc2.CParams(codec=codec))
     if aligned_chunks:
         # Aligned chunks: divisors of the shape dimensions
-        chunk_shapes = [(shape[0] // 4, shape[1] // 4) for shape in shapes]
+        chunk_shapes = [(chunks[0], chunks[1]) for shape in shapes]
     else:
         # Unaligned chunks: not divisors of shape dimensions
-        chunk_shapes = [(shape[0] // 4 + 1, shape[1] // 4 - 1) for shape in shapes]
+        chunk_shapes = [(chunks[0] + 1, chunks[1] - 1) for shape in shapes]
 
     # Create arrays
     arrays = []
     for i, (shape, chunk_shape) in enumerate(zip(shapes, chunk_shapes)):
-        arr = blosc2.arange(
-            i * np.prod(shape), (i + 1) * np.prod(shape), 1, dtype="i4", shape=shape, chunks=chunk_shape,
-            cparams=blosc2.CParams(codec=codec)
-        )
+        if datadist == "linspace":
+            # Create arrays with linearly spaced values
+            arr = blosc2.linspace(i, i + 1, num=np.prod(shape),
+                                  dtype=dtype, shape=shape, chunks=chunk_shape,
+                                  cparams=blosc2.CParams(codec=codec))
+        else:
+            # Default to arange for simplicity
+            arr = blosc2.arange(
+                i * np.prod(shape), (i + 1) * np.prod(shape), 1, dtype=dtype, shape=shape, chunks=chunk_shape,
+                cparams=blosc2.CParams(codec=codec)
+            )
         arrays.append(arr)
 
     # Calculate total data size in GB (4 bytes per int32)
@@ -67,14 +79,16 @@ def run_benchmark(num_arrays=10, size=500, aligned_chunks=False, axis=0, codec=b
     return duration, result.shape, data_size_gb
 
 
-def run_numpy_benchmark(num_arrays=10, size=500, axis=0):
+def run_numpy_benchmark(num_arrays=10, size=500, axis=0, dtype=np.float64, datadist="linspace"):
     """
     Benchmark numpy.concatenate performance for comparison.
 
     Parameters:
     - num_arrays: Number of arrays to concatenate
     - size: Base size for array dimensions
     - axis: Axis along which to concatenate (0 or 1)
+    - dtype: Data type for the arrays (default is np.float64)
+    - datadist: Distribution of data in arrays (default is "linspace")
 
     Returns:
     - duration: Time taken in seconds
@@ -93,12 +107,11 @@ def run_numpy_benchmark(num_arrays=10, size=500, axis=0):
     # Create arrays
     numpy_arrays = []
     for i, shape in enumerate(shapes):
-        arr = np.arange(
-            i * np.prod(shape),
-            (i + 1) * np.prod(shape),
-            1,
-            dtype="i4"
-        ).reshape(shape)
+        if datadist == "linspace":
+            # Create arrays with linearly spaced values
+            arr = np.linspace(i, i + 1, num=np.prod(shape), dtype=dtype).reshape(shape)
+        else:
+            arr = np.arange(i * np.prod(shape), (i + 1) * np.prod(shape), 1, dtype=dtype).reshape(shape)
         numpy_arrays.append(arr)
 
     # Calculate total data size in GB (4 bytes per int32)
@@ -114,7 +127,8 @@ def run_numpy_benchmark(num_arrays=10, size=500, axis=0):
 
 
 def create_combined_plot(num_arrays, sizes, numpy_speeds_axis0, unaligned_speeds_axis0, aligned_speeds_axis0,
-                        numpy_speeds_axis1, unaligned_speeds_axis1, aligned_speeds_axis1, output_dir="plots"):
+                         numpy_speeds_axis1, unaligned_speeds_axis1, aligned_speeds_axis1, output_dir="plots",
+                         datadist="linspace", codec_str="LZ4"):
     """
     Create a figure with two side-by-side bar plots comparing the performance for both axes.
 
@@ -148,7 +162,7 @@ def create_combined_plot(num_arrays, sizes, numpy_speeds_axis0, unaligned_speeds
     # Add labels and titles
     for ax, axis in [(ax0, 0), (ax1, 1)]:
         ax.set_xlabel('Array Size (N for NxN array)', fontsize=12)
-        ax.set_title(f'Concatenation Performance for {num_arrays} arrays (axis={axis})', fontsize=14)
+        ax.set_title(f'Concatenation Performance for {num_arrays} arrays (axis={axis}) [{datadist}, {codec_str}]', fontsize=14)
         ax.set_xticks(x)
         ax.set_xticklabels(x_labels)
         ax.grid(True, axis='y', linestyle='--', alpha=0.7)
@@ -186,22 +200,25 @@ def autolabel(rects, ax):
 
     # Save the plot
     plt.tight_layout()
-    plt.savefig(os.path.join(output_dir, 'concatenate_benchmark_combined.png'), dpi=300)
+    plt.savefig(os.path.join(output_dir, 'concatenate_benchmark_combined.png'), dpi=100)
     plt.show()
     plt.close()
 
     print(f"Combined plot saved to {os.path.join(output_dir, 'concatenate_benchmark_combined.png')}")
 
 
 def main():
-    codec = blosc2.Codec.BLOSCLZ
+    # Parameters
+    sizes = [500, 1000, 2000, 4000, 10000]  #, 20000]  # Sizes of arrays to test
+    num_arrays = 10
+    dtype = np.float64  # Data type for arrays
+    datadist = "linspace"  # Distribution of data in arrays
+    codec = blosc2.Codec.LZ4
+    codec_str = str(codec).split('.')[-1]
     print(f"{'=' * 70}")
-    print(f"Blosc2 vs NumPy concatenation benchmark {codec=}")
+    print(f"Blosc2 vs NumPy concatenation benchmark with {codec_str} codec")
     print(f"{'=' * 70}")
 
-    # Parameters
-    sizes = [500, 1000, 2000, 4000] #, 10000]  # must be divisible by 4 for aligned chunks
-    num_arrays = 10
 
     # Lists to store results for both axes
     numpy_speeds_axis0 = []
@@ -212,16 +229,18 @@ def main():
     aligned_speeds_axis1 = []
 
     for axis in [0, 1]:
-        print(f"\nConcatenating {num_arrays} arrays along axis {axis}")
+        print(f"\nConcatenating {num_arrays} arrays along axis {axis} with data distribution '{datadist}' ")
         print(f"{'Size':<8} {'NumPy (GB/s)':<14} {'Unaligned (GB/s)':<18} "
               f"{'Aligned (GB/s)':<16} {'Alig vs Unalig':<16} {'Alig vs NumPy':<16}")
         print(f"{'-' * 90}")
 
         for size in sizes:
             # Run the benchmarks
-            numpy_time, numpy_shape, data_size_gb = run_numpy_benchmark(num_arrays, size, axis=axis)
-            unaligned_time, shape1, _ = run_benchmark(num_arrays, size, aligned_chunks=False, axis=axis, codec=codec)
-            aligned_time, shape2, _ = run_benchmark(num_arrays, size, aligned_chunks=True, axis=axis, codec=codec)
+            numpy_time, numpy_shape, data_size_gb = run_numpy_benchmark(num_arrays, size, axis=axis, dtype=dtype)
+            unaligned_time, shape1, _ = run_benchmark(num_arrays, size, aligned_chunks=False, axis=axis,
+                                                      dtype=dtype, datadist=datadist, codec=codec)
+            aligned_time, shape2, _ = run_benchmark(num_arrays, size, aligned_chunks=True, axis=axis,
+                                                    dtype=dtype, datadist=datadist, codec=codec)
 
             # Calculate throughputs in GB/s
             numpy_speed = data_size_gb / numpy_time if numpy_time > 0 else float("inf")
@@ -266,7 +285,8 @@ def main():
         num_arrays,
         sizes,
         numpy_speeds_axis0, unaligned_speeds_axis0, aligned_speeds_axis0,
-        numpy_speeds_axis1, unaligned_speeds_axis1, aligned_speeds_axis1
+        numpy_speeds_axis1, unaligned_speeds_axis1, aligned_speeds_axis1,
+        datadist=datadist, output_dir="plots", codec_str=codec_str,
     )
 
 
diff --git a/bench/ndarray/concatenate_benchmark_combined.png b/bench/ndarray/concatenate_benchmark_combined.png
diff --git a/bench/ndarray/plots/concatenate_benchmark_combined-blosclz-20k.png b/bench/ndarray/plots/concatenate_benchmark_combined-blosclz-20k.png
diff --git a/bench/ndarray/plots/concatenate_benchmark_combined-lz4-20k.png b/bench/ndarray/plots/concatenate_benchmark_combined-lz4-20k.png
diff --git a/bench/ndarray/plots/concatenate_benchmark_combined-zstd-20k.png b/bench/ndarray/plots/concatenate_benchmark_combined-zstd-20k.png
diff --git a/src/blosc2/core.py b/src/blosc2/core.py
@@ -1092,9 +1092,7 @@ def detect_number_of_cores() -> int:
 # Dictionaries for the maps between compressor names and libs
 codecs = compressor_list(plugins=True)
 # Map for compression libraries and versions
-clib_versions = {}
-for codec in compressor_list(plugins=False):
-    clib_versions[codec.name] = clib_info(codec)[1].decode("utf-8")
+clib_versions = {codec.name: clib_info(codec)[1].decode("utf-8") for codec in compressor_list(plugins=False)}
 
 
 def os_release_pretty_name():
@@ -1422,15 +1420,14 @@ def compute_partition(nitems, maxshape, minpart=None):
         if rsize <= max_items:
             # rsize = rsize if size % rsize == 0 else nearest_divisor(size, rsize)
             rsize = rsize if size % rsize == 0 else blosc2_ext.nearest_divisor(size, rsize)
-            partition[-(i + 1)] = rsize
         else:
             rsize = max(max_items, minsize)
             # new_rsize = rsize if size % rsize == 0 else nearest_divisor(size, rsize, strict=True)
             new_rsize = rsize if size % rsize == 0 else blosc2_ext.nearest_divisor(size, rsize, strict=True)
             # If the new rsize is not too far from the original rsize, use it
             if rsize // 2 < new_rsize < rsize * 2:
                 rsize = new_rsize
-            partition[-(i + 1)] = rsize
+        partition[-(i + 1)] = rsize
         max_items //= rsize
 
     return partition
diff --git a/src/blosc2/helpers.py b/src/blosc2/helpers.py
@@ -32,7 +32,7 @@ def wrapper(child_func):
                     # Next parameter starts, stop copying lines
                     break
                 matching_lines.append(line)
-        assert len(matching_lines) > 0, (
+        assert matching_lines, (
             f"Could not extract the parameter {parameter} from the docstring of {parent_func.__name__}"
         )
 
diff --git a/src/blosc2/lazyexpr.py b/src/blosc2/lazyexpr.py
@@ -657,7 +657,7 @@ def conserve_functions(  # noqa: C901
     expression: str,
     operands_old: dict[str, blosc2.NDArray | blosc2.LazyExpr],
     operands_new: dict[str, blosc2.NDArray | blosc2.LazyExpr],
-) -> tuple(str, dict[str, blosc2.NDArray]):
+) -> tuple[str, dict[str, blosc2.NDArray]]:
     """
     Given an expression in string form, return its operands.
 
@@ -738,8 +738,6 @@ def visit_Name(self, node):
                     node.id = newexpr.replace(";", "")
                 else:
                     node.id = self.update_func(localop)
-            else:
-                pass
             self.generic_visit(node)
 
         def visit_Call(self, node):
@@ -784,7 +782,7 @@ def convert_to_slice(expression):
             slicer = str(slicer)
             # use slice so that lazyexpr uses blosc arrays internally
             # (and doesn't decompress according to getitem syntax)
-            new_expr += ".slice(" + slicer + ")"
+            new_expr += f".slice({slicer})"
             skip_to_char = i + k + 1
         else:
             new_expr += expr_i
@@ -833,11 +831,11 @@ def extract_numpy_scalars(expr: str):
 
 def validate_inputs(inputs: dict, out=None, reduce=False) -> tuple:  # noqa: C901
     """Validate the inputs for the expression."""
-    if len(inputs) == 0:
+    if not inputs:
         if out is None:
             raise ValueError(
                 "You really want to pass at least one input or one output for building a LazyArray."
-                "  Maybe you want blosc2.empty() instead?"
+                " Maybe you want blosc2.empty() instead?"
             )
         if isinstance(out, blosc2.NDArray):
             return out.shape, out.chunks, out.blocks, True
@@ -854,7 +852,7 @@ def validate_inputs(inputs: dict, out=None, reduce=False) -> tuple:  # noqa: C90
 
     # More checks specific of NDArray inputs
     NDinputs = [input for input in inputs if hasattr(input, "chunks")]
-    if len(NDinputs) == 0:
+    if not NDinputs:
         # All inputs are NumPy arrays, so we cannot take the fast path
         if inputs and hasattr(inputs[0], "shape"):
             shape = inputs[0].shape
@@ -895,7 +893,7 @@ def is_full_slice(item):
     elif isinstance(item, int | bool):
         return False
     else:
-        return item == slice(None, None, None) or item == Ellipsis
+        return item in (slice(None, None, None), Ellipsis)
 
 
 def do_slices_intersect(slice1: list | tuple, slice2: list | tuple) -> bool:
@@ -1925,7 +1923,7 @@ def fuse_expressions(expr, new_base, dup_op):
         if i < skip_to_char:
             continue
         if expr_i == "o":
-            if i > 0 and (expr[i - 1] != " " and expr[i - 1] != "("):
+            if i > 0 and expr[i - 1] not in {" ", "("}:
                 # Not a variable
                 new_expr += expr_i
                 continue
@@ -2960,18 +2958,18 @@ def info(self):
 
     @property
     def info_items(self):
-        items = []
-        items += [("type", f"{self.__class__.__name__}")]
         inputs = {}
         for key, value in self.inputs_dict.items():
             if isinstance(value, np.ndarray | blosc2.NDArray | blosc2.C2Array):
                 inputs[key] = f"<{value.__class__.__name__}> {value.shape} {value.dtype}"
             else:
                 inputs[key] = str(value)
-        items += [("inputs", inputs)]
-        items += [("shape", self.shape)]
-        items += [("dtype", self.dtype)]
-        return items
+        return [
+            ("type", f"{self.__class__.__name__}"),
+            ("inputs", inputs),
+            ("shape", self.shape),
+            ("dtype", self.dtype),
+        ]
 
     # TODO: indices and sort are repeated in LazyExpr; refactor
     def indices(self, order: str | list[str] | None = None) -> blosc2.LazyArray:
diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py
@@ -3524,7 +3524,7 @@ def copy(array: NDArray, dtype: np.dtype | str = None, **kwargs: Any) -> NDArray
 
 
 def concatenate(arrays: list[NDArray], /, axis=0, **kwargs: Any) -> NDArray:  # noqa: C901
-    """Concatenate two arrays along a specified axis.
+    """Concatenate a list of arrays along a specified axis.
 
     Parameters
     ----------
@@ -3755,8 +3755,8 @@ def _check_ndarray_kwargs(**kwargs):  # noqa: C901
     else:
         # Add the default storage values as long as they are not already passed
         storage_dflts = asdict(blosc2.Storage(urlpath=kwargs.get("urlpath")))  # urlpath can affect defaults
-        not_passed = {k: v for k, v in storage_dflts.items() if k not in kwargs}
-        kwargs = {**kwargs, **not_passed}
+        # If a key appears in both operands, the one from the right-hand operand wins
+        kwargs = storage_dflts | kwargs
 
     supported_keys = [
         "chunks",
diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py
@@ -731,15 +731,11 @@ def apply(cls, data, func, args, kwargs, decorator, axis):
             return func(data, *args, **kwargs)
         elif axis in (0, "index"):
             # pandas apply(axis=0) column-wise
-            result = []
-            for row_idx in range(data.shape[1]):
-                result.append(func(data[:, row_idx], *args, **kwargs))
+            result = [func(data[:, row_idx], *args, **kwargs) for row_idx in range(data.shape[1])]
             return np.vstack(result).transpose()
         elif axis in (1, "columns"):
             # pandas apply(axis=1) row-wise
-            result = []
-            for col_idx in range(data.shape[0]):
-                result.append(func(data[col_idx, :], *args, **kwargs))
+            result = [func(data[col_idx, :], *args, **kwargs) for col_idx in range(data.shape[0])]
             return np.vstack(result)
         else:
             raise NotImplementedError(f"Unknown axis '{axis}'. Use one of 0, 1 or None.")
diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py
@@ -68,8 +68,7 @@ def __len__(self):
         return super().nvlmetalayers()
 
     def __iter__(self):
-        keys = super().get_names()
-        yield from keys
+        yield from super().get_names()
 
     def getall(self):
         """
diff --git a/tests/ndarray/test_lossy.py b/tests/ndarray/test_lossy.py
@@ -75,11 +75,10 @@ def test_lossy(shape, cparams, dtype, urlpath, contiguous):
         or a.schunk.cparams.filters[0] == blosc2.Filter.NDMEAN
     ):
         _ = a[...]
-    else:
+    elif dtype in (np.float32, np.float64):
         tol = 1e-5
-        if dtype in (np.float32, np.float64):
-            np.testing.assert_allclose(a[...], array, rtol=tol, atol=tol)
-        else:
-            np.array_equal(a[...], array)
+        np.testing.assert_allclose(a[...], array, rtol=tol, atol=tol)
+    else:
+        np.array_equal(a[...], array)
 
     blosc2.remove_urlpath(urlpath)
diff --git a/tests/test_schunk_constructor.py b/tests/test_schunk_constructor.py
@@ -158,7 +158,7 @@ def test_schunk_fill_special(contiguous, urlpath, cparams, nitems, special_value
         if isinstance(expected_value, float):
             dtype = np.float32
         elif isinstance(expected_value, bytes):
-            dtype = np.dtype("|S" + str(len(expected_value)))
+            dtype = np.dtype(f"|S{len(expected_value)}")
         array = np.full(nitems, expected_value, dtype=dtype)
         dest = np.empty(nitems, dtype=dtype)
         schunk.get_slice(out=dest)

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ def wrapper(child_func):`
`32`	`32`	`# Next parameter starts, stop copying lines`
`33`	`33`	`break`
`34`	`34`	`matching_lines.append(line)`
`35`		`- assert len(matching_lines) > 0, (`
	`35`	`+ assert matching_lines, (`
`36`	`36`	`f"Could not extract the parameter {parameter} from the docstring of {parent_func.__name__}"`
`37`	`37`	`)`
`38`	`38`