Merge branch 'ctable4' of github.com:Blosc/python-blosc2 into ctable4

Author: FrancescAlted

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.4
+    rev: v0.15.9
     hooks:
       - id: ruff-check
         args: ["--fix", "--show-fixes"]

AGENTS.md

@@ -0,0 +1,53 @@
+# Repository Guidelines
+
+## Project Structure & Module Organization
+The Python package lives in `src/blosc2/`, including the C/Cython extension sources
+(`blosc2_ext.*`) and core modules such as `core.py`, `ndarray.py`, and `schunk.py`.
+Tests are under `tests/`, with additional doctests enabled for select modules per
+`pytest.ini`. Documentation sources are in `doc/` and build output lands in `html/`.
+Examples are in `examples/`, and performance/benchmark scripts live in `bench/`.
+
+## Build, Test, and Development Commands
+- `pip install .` builds the bundled C-Blosc2 and installs the package.
+- `pip install -e .` installs in editable mode for local development.
+- `CMAKE_PREFIX_PATH=/usr/local USE_SYSTEM_BLOSC2=1 pip install -e .` builds
+  against a separately installed C-Blosc2.
+- `pytest` runs the default test suite (excludes `heavy` and `network` markers).
+- `pytest -m "heavy"` runs long-running tests.
+- `pytest -m "network"` runs tests requiring network access.
+- `cd doc && rm -rf ../html _build && python -m sphinx . ../html` builds docs.
+
+## Coding Style & Naming Conventions
+Use Ruff for formatting and linting (line length 109). Enable pre-commit hooks:
+`python -m pip install pre-commit` then `pre-commit install`. Follow Python
+conventions: 4-space indentation, `snake_case` for functions/variables, and
+`PascalCase` for classes. Pytest discovery expects `tests/test_*.py` and
+`test_*` functions. Do not use leading underscores in module-level helper
+function names when those helpers are imported from other modules; reserve
+leading underscores for file-local implementation details. Avoid leading
+underscores in core module filenames under `src/blosc2/`; prefer non-underscored
+module names unless there is a strong reason to keep a module private.
+
+For documentation and tutorial query examples, prefer the shortest idiom that
+matches the intended result type. Use `expr[:]` or `arr[mask][:]` when showing
+values, use `expr.compute()` when materializing an `NDArray`, and use
+`expr.compute(_use_index=False)` when demonstrating scan-vs-index behavior.
+Avoid `expr.compute()[:]` unless a NumPy array is specifically required.
+
+## Testing Guidelines
+Pytest is required; warnings are treated as errors. The default configuration
+adds `--doctest-modules`, so keep doctest examples in `blosc2/core.py`,
+`blosc2/ndarray.py`, and `blosc2/schunk.py` accurate. Use markers `heavy` and
+`network` for slow or network-dependent tests.
+
+## Commit & Pull Request Guidelines
+Recent commit messages are short, imperative sentences (e.g., “Add …”, “Fix …”)
+without ticket prefixes. For pull requests: branch from `main`, add tests for
+behavior changes, update docs for API changes, ensure the test suite passes,
+and avoid introducing new compiler warnings. Link issues when applicable and
+include clear reproduction steps for bug fixes.
+
+**IMPORTANT — Agent commit policy**: Never run `git commit`, `git push`,
+`git reset`, `git rebase`, or any other destructive/state-changing git operation.
+Do not ask for permission to do so either. Committing and all repo state changes
+are exclusively the user’s decision. Just do the code work and stop there.

CMakeLists.txt

@@ -41,11 +41,25 @@ add_custom_command(
   DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx"
   VERBATIM)
 
+add_custom_command(
+  OUTPUT indexing_ext.c
+  COMMAND Python::Interpreter -m cython
+          "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/indexing_ext.pyx" --output-file indexing_ext.c
+  DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/indexing_ext.pyx"
+  VERBATIM)
+
 # ...and add it to the target
 Python_add_library(blosc2_ext MODULE blosc2_ext.c WITH_SOABI)
+target_sources(blosc2_ext PRIVATE src/blosc2/matmul_kernels.c)
+target_include_directories(blosc2_ext PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2)
+if(UNIX)
+  target_link_libraries(blosc2_ext PRIVATE ${CMAKE_DL_LIBS})
+endif()
+Python_add_library(indexing_ext MODULE indexing_ext.c WITH_SOABI)
 
 # We need to link against NumPy
 target_link_libraries(blosc2_ext PRIVATE Python::NumPy)
+target_link_libraries(indexing_ext PRIVATE Python::NumPy)
 
 # Fetch and build miniexpr library
 include(FetchContent)
@@ -63,15 +77,19 @@ endif()
 
 FetchContent_Declare(miniexpr
     GIT_REPOSITORY https://github.com/Blosc/miniexpr.git
-    GIT_TAG feadbc633a887bafd84b2fbc370ef2962d01b7ee
+    GIT_TAG f2faef741c4c507bf6a03167c72ce7f92c6f0ae8
     # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr
 )
 FetchContent_MakeAvailable(miniexpr)
 
 # Link against miniexpr static library
 target_link_libraries(blosc2_ext PRIVATE miniexpr_static)
+if(APPLE)
+  target_link_libraries(blosc2_ext PRIVATE "-framework Accelerate")
+endif()
 
 target_compile_features(blosc2_ext PRIVATE c_std_11)
+target_compile_features(indexing_ext PRIVATE c_std_11)
 if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
     execute_process(
         COMMAND "${CMAKE_C_COMPILER}" -print-resource-dir
@@ -119,7 +137,7 @@ else()
     include(FetchContent)
     FetchContent_Declare(blosc2
         GIT_REPOSITORY https://github.com/Blosc/c-blosc2
-        GIT_TAG b32256fc1287b6e24c22f09ac202265c7054e2bc
+        GIT_TAG 2fdba83b4046f22fed95a794d43a0376cce3d5db
         # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../c-blosc2
     )
     FetchContent_MakeAvailable(blosc2)
@@ -148,7 +166,7 @@ endif()
 
 # Python extension -> site-packages/blosc2
 install(
-  TARGETS blosc2_ext
+  TARGETS blosc2_ext indexing_ext
   LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/blosc2
 )
 

README_DEVELOPERS.md

@@ -93,6 +93,36 @@ If you want to run the network tests, you can use the following command:
   pytest -m "network"
 ```
 
+## Matmul backend discovery
+
+The fast `blosc2.matmul` path uses platform-specific block kernels:
+
+- macOS: `Accelerate`
+- Linux/Windows: runtime-discovered `cblas`
+- fallback: portable `naive` kernel
+
+For the runtime `cblas` backend, `python-blosc2` probes the active Python/NumPy
+environment rather than linking to one BLAS vendor at build time.  Discovery
+starts from NumPy's reported BLAS library directory when available, and then
+searches common library names in the active environment's `lib` directories.
+
+On Linux the current candidates include `libcblas`, `libopenblas`,
+`libflexiblas`, `libblis`, `libmkl_rt`, and generic `libblas`.  A candidate is
+accepted only if it loads successfully and exports both `cblas_sgemm` and
+`cblas_dgemm`.  If no suitable provider is found, the fast path falls back to
+the `naive` kernel.
+
+Useful runtime helpers:
+
+- `blosc2.get_matmul_library()` reports the selected runtime library when available
+- `BLOSC_TRACE=1` logs candidate probing, rejection, selection, and backend fallback
+
+Example:
+
+```bash
+BLOSC_TRACE=1 python -c "import blosc2; print(blosc2.get_matmul_library())"
+```
+
 ## wasm32 / Pyodide developer workflow
 
 For the local wasm32 workflow (uv + pyodide-build + cibuildwheel + test loop),

bench/batch_store.py bench/batch_array.pybench/batch_store.py renamed to bench/batch_array.py

@@ -5,6 +5,10 @@
 # SPDX-License-Identifier: BSD-3-Clause
 #######################################################################
 
+# This benchmarks BatchArray random single-item reads. It supports
+# msgpack or arrow, configurable codec/compression level, optional
+# dictionary compression, and in-memory vs persistent mode.
+
 from __future__ import annotations
 
 import argparse
@@ -15,7 +19,7 @@
 import blosc2
 
 
-URLPATH = "bench_batch_store.b2b"
+URLPATH = "bench_batch_array.b2b"
 NBATCHES = 10_000
 OBJECTS_PER_BATCH = 100
 TOTAL_OBJECTS = NBATCHES * OBJECTS_PER_BATCH
@@ -46,23 +50,23 @@ def expected_entry(batch_index: int, item_index: int) -> dict[str, int]:
 
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
-        description="Benchmark BatchStore single-entry reads.",
+        description="Benchmark BatchArray single-entry reads.",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     parser.add_argument("--codec", type=str, default="ZSTD", choices=[codec.name for codec in blosc2.Codec])
     parser.add_argument("--clevel", type=int, default=5)
     parser.add_argument("--serializer", type=str, default="msgpack", choices=["msgpack", "arrow"])
     parser.add_argument("--use-dict", action="store_true", help="Enable dictionaries for ZSTD/LZ4/LZ4HC codecs.")
-    parser.add_argument("--in-mem", action="store_true", help="Keep the BatchStore purely in memory.")
+    parser.add_argument("--in-mem", action="store_true", help="Keep the BatchArray purely in memory.")
     return parser
 
 
-def build_store(
+def build_array(
     codec: blosc2.Codec, clevel: int, use_dict: bool, serializer: str, in_mem: bool
-) -> blosc2.BatchStore | None:
+) -> blosc2.BatchArray | None:
     if in_mem:
         storage = blosc2.Storage(mode="w")
-        store = blosc2.BatchStore(
+        barr = blosc2.BatchArray(
             storage=storage,
             items_per_block=ITEMS_PER_BLOCK,
             serializer=serializer,
@@ -73,8 +77,8 @@ def build_store(
             },
         )
         for batch_index in range(NBATCHES):
-            store.append(make_batch(batch_index))
-        return store
+            barr.append(make_batch(batch_index))
+        return barr
 
     blosc2.remove_urlpath(URLPATH)
     storage = blosc2.Storage(urlpath=URLPATH, mode="w", contiguous=True)
@@ -83,24 +87,24 @@ def build_store(
         "clevel": clevel,
         "use_dict": use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4, blosc2.Codec.LZ4HC),
     }
-    with blosc2.BatchStore(
+    with blosc2.BatchArray(
         storage=storage, items_per_block=ITEMS_PER_BLOCK, serializer=serializer, cparams=cparams
-    ) as store:
+    ) as barr:
         for batch_index in range(NBATCHES):
-            store.append(make_batch(batch_index))
+            barr.append(make_batch(batch_index))
     return None
 
 
-def measure_random_reads(store: blosc2.BatchStore) -> tuple[list[tuple[int, int, int, dict[str, int]]], list[int]]:
+def measure_random_reads(barr: blosc2.BatchArray) -> tuple[list[tuple[int, int, int, dict[str, int]]], list[int]]:
     rng = random.Random(2024)
     samples: list[tuple[int, int, int, dict[str, int]]] = []
     timings_ns: list[int] = []
 
     for _ in range(N_RANDOM_READS):
-        batch_index = rng.randrange(len(store))
+        batch_index = rng.randrange(len(barr))
         item_index = rng.randrange(OBJECTS_PER_BATCH)
         t0 = time.perf_counter_ns()
-        value = store[batch_index][item_index]
+        value = barr[batch_index][item_index]
         timings_ns.append(time.perf_counter_ns() - t0)
         if value != expected_entry(batch_index, item_index):
             raise RuntimeError(f"Value mismatch at batch={batch_index}, item={item_index}")
@@ -117,39 +121,39 @@ def main() -> None:
 
     mode_label = "in-memory" if args.in_mem else "persistent"
     article = "an" if args.in_mem else "a"
-    print(f"Building {article} {mode_label} BatchStore with 1,000,000 RGB dicts and timing 1,000 random scalar reads...")
+    print(f"Building {article} {mode_label} BatchArray with 1,000,000 RGB dicts and timing 1,000 random scalar reads...")
     print(f"  codec: {codec.name}")
     print(f"  clevel: {args.clevel}")
     print(f"  serializer: {args.serializer}")
     print(f"  use_dict: {use_dict}")
     print(f"  in_mem: {args.in_mem}")
     t0 = time.perf_counter()
-    store = build_store(
+    barr = build_array(
         codec=codec, clevel=args.clevel, use_dict=use_dict, serializer=args.serializer, in_mem=args.in_mem
     )
     build_time_s = time.perf_counter() - t0
     if args.in_mem:
-        assert store is not None
-        read_store = store
+        assert barr is not None
+        read_array = barr
     else:
-        read_store = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, items_per_block=ITEMS_PER_BLOCK)
-    samples, timings_ns = measure_random_reads(read_store)
+        read_array = blosc2.BatchArray(urlpath=URLPATH, mode="r", contiguous=True, items_per_block=ITEMS_PER_BLOCK)
+    samples, timings_ns = measure_random_reads(read_array)
     t0 = time.perf_counter()
     checksum = 0
     nitems = 0
-    for item in read_store.iter_items():
+    for item in read_array.iter_items():
         checksum += item["blue"]
         nitems += 1
     iter_time_s = time.perf_counter() - t0
 
     print()
-    print("BatchStore benchmark")
+    print("BatchArray benchmark")
     print(f"  build time: {build_time_s:.3f} s")
-    print(f"  batches: {len(read_store)}")
+    print(f"  batches: {len(read_array)}")
     print(f"  items: {TOTAL_OBJECTS}")
-    print(f"  items_per_block: {read_store.items_per_block}")
+    print(f"  items_per_block: {read_array.items_per_block}")
     print()
-    print(read_store.info)
+    print(read_array.info)
     print(f"Random scalar reads: {N_RANDOM_READS}")
     print(f"  mean: {statistics.fmean(timings_ns) / 1_000:.2f} us")
     print(f"  max:  {max(timings_ns) / 1_000:.2f} us")
@@ -159,11 +163,11 @@ def main() -> None:
     print(f"  checksum: {checksum}")
     print("Sample reads:")
     for timing_ns, batch_index, item_index, value in samples[:5]:
-        print(f"  {timing_ns / 1_000:.2f} us -> read_store[{batch_index}][{item_index}] = {value}")
+        print(f"  {timing_ns / 1_000:.2f} us -> read_array[{batch_index}][{item_index}] = {value}")
     if args.in_mem:
-        print("BatchStore kept in memory")
+        print("BatchArray kept in memory")
     else:
-        print(f"BatchStore file at: {read_store.urlpath}")
+        print(f"BatchArray file at: {read_array.urlpath}")
 
 
 if __name__ == "__main__":