blocksize_max -> max_blocksize. also, this is persisted in metalayer now.

Author: FrancescAlted

bench/batch_store.py

@@ -61,7 +61,7 @@ def build_store(codec: blosc2.Codec, clevel: int, use_dict: bool, in_mem: bool)
         storage = blosc2.Storage(mode="w")
         store = blosc2.BatchStore(
             storage=storage,
-            blocksize_max=BLOCKSIZE_MAX,
+            max_blocksize=BLOCKSIZE_MAX,
             cparams={
                 "codec": codec,
                 "clevel": clevel,
@@ -79,7 +79,7 @@ def build_store(codec: blosc2.Codec, clevel: int, use_dict: bool, in_mem: bool)
         "clevel": clevel,
         "use_dict": use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4),
     }
-    with blosc2.BatchStore(storage=storage, blocksize_max=BLOCKSIZE_MAX, cparams=cparams) as store:
+    with blosc2.BatchStore(storage=storage, max_blocksize=BLOCKSIZE_MAX, cparams=cparams) as store:
         for batch_index in range(NBATCHES):
             store.append(make_batch(batch_index))
     return None
@@ -123,7 +123,7 @@ def main() -> None:
         assert store is not None
         read_store = store
     else:
-        read_store = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, blocksize_max=BLOCKSIZE_MAX)
+        read_store = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, max_blocksize=BLOCKSIZE_MAX)
     samples, timings_ns = measure_random_reads(read_store)
     t0 = time.perf_counter()
     checksum = 0
@@ -138,7 +138,7 @@ def main() -> None:
     print(f"  build time: {build_time_s:.3f} s")
     print(f"  batches: {len(read_store)}")
     print(f"  objects: {TOTAL_OBJECTS}")
-    print(f"  blocksize_max: {read_store.blocksize_max}")
+    print(f"  max_blocksize: {read_store.max_blocksize}")
     print()
     print(read_store.info)
     print(f"Random scalar reads: {N_RANDOM_READS}")

examples/batch_store.py

@@ -36,19 +36,19 @@ def main() -> None:
     blosc2.remove_urlpath(URLPATH)
 
     storage = blosc2.Storage(urlpath=URLPATH, mode="w", contiguous=True)
-    with blosc2.BatchStore(storage=storage, blocksize_max=BLOCKSIZE_MAX) as store:
+    with blosc2.BatchStore(storage=storage, max_blocksize=BLOCKSIZE_MAX) as store:
         for batch_index in range(NBATCHES):
             store.append(make_batch(batch_index))
 
         total_objects = sum(len(batch) for batch in store)
         print("Created BatchStore")
         print(f"  batches: {len(store)}")
         print(f"  objects: {total_objects}")
-        print(f"  blocksize_max: {store.blocksize_max}")
+        print(f"  max_blocksize: {store.max_blocksize}")
 
-    # Reopen with the same blocksize_max hint so scalar reads can use the
+    # Reopen with the same max_blocksize hint so scalar reads can use the
     # VL-block path instead of decoding the entire batch.
-    reopened = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, blocksize_max=BLOCKSIZE_MAX)
+    reopened = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, max_blocksize=BLOCKSIZE_MAX)
 
     print()
     print(reopened.info)

src/blosc2/batch_store.py

@@ -18,7 +18,7 @@
 from blosc2._msgpack_utils import msgpack_packb, msgpack_unpackb
 from blosc2.info import InfoReporter, format_nbytes_info
 
-_BATCHSTORE_META = {"version": 1, "serializer": "msgpack"}
+_BATCHSTORE_META = {"version": 1, "serializer": "msgpack", "max_blocksize": None}
 
 
 def _check_serialized_size(buffer: bytes) -> None:
@@ -69,9 +69,9 @@ def __getitem__(self, index: int | slice) -> Any | list[Any]:
             items = self._decode_items()
             index = self._normalize_index(index)
             return items[index]
-        blocksize_max = self._parent.blocksize_max
-        if blocksize_max is not None:
-            block_index, item_index = divmod(index, blocksize_max)
+        max_blocksize = self._parent.max_blocksize
+        if max_blocksize is not None:
+            block_index, item_index = divmod(index, max_blocksize)
             if block_index >= self._nblocks:
                 raise IndexError("Batch index out of range")
             block = self._get_block(block_index)
@@ -158,6 +158,11 @@ def _attach_schunk(self, schunk: blosc2.SChunk) -> None:
         self.schunk = schunk
         self.mode = schunk.mode
         self.mmap_mode = getattr(schunk, "mmap_mode", None)
+        try:
+            batchstore_meta = self.schunk.meta["batchstore"]
+        except KeyError:
+            batchstore_meta = {}
+        self._max_blocksize = batchstore_meta.get("max_blocksize", self._max_blocksize)
         self._validate_tag()
 
     def _maybe_open_existing(self, storage: blosc2.Storage) -> bool:
@@ -181,13 +186,13 @@ def _make_storage(self) -> blosc2.Storage:
 
     def __init__(
         self,
-        blocksize_max: int | None = None,
+        max_blocksize: int | None = None,
         _from_schunk: blosc2.SChunk | None = None,
         **kwargs: Any,
     ) -> None:
-        if blocksize_max is not None and blocksize_max <= 0:
-            raise ValueError("blocksize_max must be a positive integer")
-        self._blocksize_max: int | None = blocksize_max
+        if max_blocksize is not None and max_blocksize <= 0:
+            raise ValueError("max_blocksize must be a positive integer")
+        self._max_blocksize: int | None = max_blocksize
         if _from_schunk is not None:
             if kwargs:
                 unexpected = ", ".join(sorted(kwargs))
@@ -213,7 +218,7 @@ def __init__(
             return
 
         fixed_meta = dict(storage.meta or {})
-        fixed_meta["batchstore"] = dict(_BATCHSTORE_META)
+        fixed_meta["batchstore"] = {**_BATCHSTORE_META, "max_blocksize": self._max_blocksize}
         storage.meta = fixed_meta
         schunk = blosc2.SChunk(chunksize=-1, data=None, cparams=cparams, dparams=dparams, storage=storage)
         self._attach_schunk(schunk)
@@ -263,9 +268,29 @@ def _normalize_batch(self, value: object) -> list[Any]:
         return values
 
     def _ensure_layout_for_batch(self, batch: list[Any]) -> None:
-        if self._blocksize_max is None:
+        if self._max_blocksize is None:
             payload_sizes = [len(msgpack_packb(item)) for item in batch]
-            self._blocksize_max = self._guess_blocksize(payload_sizes)
+            self._max_blocksize = self._guess_blocksize(payload_sizes)
+            self._persist_max_blocksize()
+
+    def _persist_max_blocksize(self) -> None:
+        if self._max_blocksize is None or len(self) > 0:
+            return
+        storage = self._make_storage()
+        fixed_meta = dict(storage.meta or {})
+        fixed_meta["batchstore"] = {
+            **dict(fixed_meta.get("batchstore", {})),
+            "max_blocksize": self._max_blocksize,
+        }
+        storage.meta = fixed_meta
+        schunk = blosc2.SChunk(
+            chunksize=-1,
+            data=None,
+            cparams=copy.deepcopy(self.cparams),
+            dparams=copy.deepcopy(self.dparams),
+            storage=storage,
+        )
+        self._attach_schunk(schunk)
 
     def _guess_blocksize(self, payload_sizes: list[int]) -> int:
         if not payload_sizes:
@@ -301,11 +326,11 @@ def _vl_dparams_kwargs(self) -> dict[str, Any]:
         return asdict(self.schunk.dparams)
 
     def _compress_batch(self, batch: list[Any]) -> bytes:
-        if self._blocksize_max is None:
-            raise RuntimeError("BatchStore blocksize_max is not initialized")
+        if self._max_blocksize is None:
+            raise RuntimeError("BatchStore max_blocksize is not initialized")
         blocks = [
-            self._serialize_block(batch[i : i + self._blocksize_max])
-            for i in range(0, len(batch), self._blocksize_max)
+            self._serialize_block(batch[i : i + self._max_blocksize])
+            for i in range(0, len(batch), self._max_blocksize)
         ]
         return blosc2.blosc2_ext.vlcompress(blocks, **self._vl_cparams_kwargs())
 
@@ -446,8 +471,8 @@ def dparams(self):
         return self.schunk.dparams
 
     @property
-    def blocksize_max(self) -> int | None:
-        return self._blocksize_max
+    def max_blocksize(self) -> int | None:
+        return self._max_blocksize
 
     @property
     def typesize(self) -> int:
@@ -492,7 +517,7 @@ def info_items(self) -> list:
             ("type", f"{self.__class__.__name__}"),
             ("nbatches", len(self)),
             ("batch stats", batch_stats),
-            ("blocksize_max", self.blocksize_max),
+            ("max_blocksize", self.max_blocksize),
             ("nitems", sum(batch_sizes)),
             ("nbytes", format_nbytes_info(self.nbytes)),
             ("cbytes", format_nbytes_info(self.cbytes)),
@@ -510,7 +535,7 @@ def copy(self, **kwargs: Any) -> BatchStore:
             raise ValueError("meta should not be passed to copy")
         kwargs["cparams"] = kwargs.get("cparams", copy.deepcopy(self.cparams))
         kwargs["dparams"] = kwargs.get("dparams", copy.deepcopy(self.dparams))
-        kwargs["blocksize_max"] = kwargs.get("blocksize_max", self.blocksize_max)
+        kwargs["max_blocksize"] = kwargs.get("max_blocksize", self.max_blocksize)
 
         if "storage" not in kwargs:
             kwargs["meta"] = self._copy_meta()

tests/test_batch_store.py

@@ -46,8 +46,8 @@ def test_batchstore_roundtrip(contiguous, urlpath):
         assert barray.append(batch) == i
 
     assert len(barray) == len(BATCHES)
-    assert barray.blocksize_max is not None
-    assert 1 <= barray.blocksize_max <= len(BATCHES[0])
+    assert barray.max_blocksize is not None
+    assert 1 <= barray.max_blocksize <= len(BATCHES[0])
     assert [batch[:] for batch in barray.iter_batches()] == BATCHES
     assert barray.append([1, 2]) == len(BATCHES) + 1
     assert [batch[:] for batch in barray.iter_batches()][-1] == [1, 2]
@@ -83,7 +83,7 @@ def test_batchstore_roundtrip(contiguous, urlpath):
     if urlpath is not None:
         reopened = blosc2.open(urlpath, mode="r")
         assert isinstance(reopened, blosc2.BatchStore)
-        assert reopened.blocksize_max is None
+        assert reopened.max_blocksize == barray.max_blocksize
         assert [batch[:] for batch in reopened.iter_batches()] == expected
         with pytest.raises(ValueError):
             reopened.append(["nope"])
@@ -145,7 +145,7 @@ def test_batchstore_info():
     assert items["type"] == "BatchStore"
     assert items["nbatches"] == len(BATCHES)
     assert items["batch stats"].startswith("mean=")
-    assert items["blocksize_max"] == barray.blocksize_max
+    assert items["max_blocksize"] == barray.max_blocksize
     assert items["nitems"] == sum(len(batch) for batch in BATCHES)
     assert "urlpath" not in items
     assert "contiguous" not in items
@@ -158,7 +158,7 @@ def test_batchstore_info():
     assert "type" in text
     assert "BatchStore" in text
     assert "batch stats" in text
-    assert "blocksize_max" in text
+    assert "max_blocksize" in text
 
 
 def test_batchstore_zstd_does_not_use_dict_by_default():
@@ -167,9 +167,9 @@ def test_batchstore_zstd_does_not_use_dict_by_default():
     assert barray.cparams.use_dict is False
 
 
-def test_batchstore_explicit_blocksize_max():
-    barray = blosc2.BatchStore(blocksize_max=2)
-    assert barray.blocksize_max == 2
+def test_batchstore_explicit_max_blocksize():
+    barray = blosc2.BatchStore(max_blocksize=2)
+    assert barray.max_blocksize == 2
     barray.append([1, 2, 3])
     barray.append([4])
     assert [batch[:] for batch in barray.iter_batches()] == [[1, 2, 3], [4]]
@@ -180,10 +180,10 @@ def test_batchstore_get_vlblock_and_scalar_access():
     blosc2.remove_urlpath(urlpath)
 
     batch = [0, 1, 2, 3, 4]
-    barray = blosc2.BatchStore(storage=_storage(True, urlpath), blocksize_max=2)
+    barray = blosc2.BatchStore(storage=_storage(True, urlpath), max_blocksize=2)
     barray.append(batch)
 
-    assert barray.blocksize_max == 2
+    assert barray.max_blocksize == 2
     assert msgpack_unpackb(barray.schunk.get_vlblock(0, 0)) == batch[:2]
     assert msgpack_unpackb(barray.schunk.get_vlblock(0, 1)) == batch[2:4]
     assert msgpack_unpackb(barray.schunk.get_vlblock(0, 2)) == batch[4:]
@@ -194,6 +194,7 @@ def test_batchstore_get_vlblock_and_scalar_access():
 
     reopened = blosc2.open(urlpath, mode="r")
     assert isinstance(reopened, blosc2.BatchStore)
+    assert reopened.max_blocksize == 2
     assert reopened[0][0] == 0
     assert reopened[0][2] == 2
     assert reopened[0][4] == 4
@@ -203,7 +204,7 @@ def test_batchstore_get_vlblock_and_scalar_access():
 
 
 def test_batchstore_scalar_reads_cache_vlblocks():
-    barray = blosc2.BatchStore(blocksize_max=2)
+    barray = blosc2.BatchStore(max_blocksize=2)
     barray.append([0, 1, 2, 3, 4])
 
     batch = barray[0]
@@ -227,7 +228,7 @@ def wrapped_get_vlblock(nchunk, nblock):
 
 
 def test_batchstore_iter_objects():
-    barray = blosc2.BatchStore(blocksize_max=2)
+    barray = blosc2.BatchStore(max_blocksize=2)
     batches = [[1, 2, 3], [4], [5, 6]]
     barray.extend(batches)
 

tests/test_tree_store.py

@@ -658,7 +658,7 @@ def test_external_batchstore_support(tmp_path):
     store_path = tmp_path / "test_batchstore_external.b2d"
 
     with TreeStore(str(store_path), mode="w", threshold=0) as tstore:
-        bstore = blosc2.BatchStore(blocksize_max=2)
+        bstore = blosc2.BatchStore(max_blocksize=2)
         bstore.extend([[{"id": 1}, {"id": 2}], [{"id": 3}]])
         tstore["/data/batchstore"] = bstore