Adapt max_blocksize depending on the clevel

Author: FrancescAlted

bench/batch_store.py

@@ -51,7 +51,7 @@ def build_parser() -> argparse.ArgumentParser:
     )
     parser.add_argument("--codec", type=str, default="ZSTD", choices=[codec.name for codec in blosc2.Codec])
     parser.add_argument("--clevel", type=int, default=5)
-    parser.add_argument("--use-dict", action="store_true", help="Enable dictionaries for ZSTD/LZ4 codecs.")
+    parser.add_argument("--use-dict", action="store_true", help="Enable dictionaries for ZSTD/LZ4/LZ4HC codecs.")
     parser.add_argument("--in-mem", action="store_true", help="Keep the BatchStore purely in memory.")
     return parser
 
@@ -65,7 +65,7 @@ def build_store(codec: blosc2.Codec, clevel: int, use_dict: bool, in_mem: bool)
             cparams={
                 "codec": codec,
                 "clevel": clevel,
-                "use_dict": use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4),
+                "use_dict": use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4, blosc2.Codec.LZ4HC),
             },
         )
         for batch_index in range(NBATCHES):
@@ -77,7 +77,7 @@ def build_store(codec: blosc2.Codec, clevel: int, use_dict: bool, in_mem: bool)
     cparams = {
         "codec": codec,
         "clevel": clevel,
-        "use_dict": use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4),
+        "use_dict": use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4, blosc2.Codec.LZ4HC),
     }
     with blosc2.BatchStore(storage=storage, max_blocksize=BLOCKSIZE_MAX, cparams=cparams) as store:
         for batch_index in range(NBATCHES):
@@ -107,7 +107,7 @@ def main() -> None:
     parser = build_parser()
     args = parser.parse_args()
     codec = blosc2.Codec[args.codec]
-    use_dict = args.use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4)
+    use_dict = args.use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4, blosc2.Codec.LZ4HC)
 
     mode_label = "in-memory" if args.in_mem else "persistent"
     article = "an" if args.in_mem else "a"

src/blosc2/batch_store.py

@@ -295,13 +295,21 @@ def _persist_max_blocksize(self) -> None:
     def _guess_blocksize(self, payload_sizes: list[int]) -> int:
         if not payload_sizes:
             raise ValueError("BatchStore entries cannot be empty")
-        l2_cache_size = blosc2.cpu_info.get("l2_cache_size")
-        if not isinstance(l2_cache_size, int) or l2_cache_size <= 0:
+        clevel = self.cparams.clevel
+        if clevel == 9:
+            return len(payload_sizes)
+        if 0 < clevel < 6:
+            budget = blosc2.cpu_info.get("l1_data_cache_size")
+        elif 6 <= clevel < 9:
+            budget = blosc2.cpu_info.get("l2_cache_size")
+        else:
+            return len(payload_sizes)
+        if not isinstance(budget, int) or budget <= 0:
             return len(payload_sizes)
         total = 0
         count = 0
         for payload_size in payload_sizes:
-            if count > 0 and total + payload_size > l2_cache_size:
+            if count > 0 and total + payload_size > budget:
                 break
             total += payload_size
             count += 1

src/blosc2/storage.py

@@ -46,7 +46,9 @@ class CParams:
         (maximum compression). Default is 1.
     use_dict: bool
         Whether to use dictionaries when compressing
-        (only for :py:obj:`blosc2.Codec.ZSTD <Codec>`). Default is `False`.
+        (supported for :py:obj:`blosc2.Codec.ZSTD <Codec>`,
+        :py:obj:`blosc2.Codec.LZ4 <Codec>`, and
+        :py:obj:`blosc2.Codec.LZ4HC <Codec>`). Default is `False`.
     typesize: int
         The data type size, ranging from 1 to 255. Default is 8.
     nthreads: int

tests/test_batch_store.py

@@ -241,6 +241,10 @@ def test_batchstore_respects_explicit_use_dict_and_non_zstd():
     assert barray.cparams.codec == blosc2.Codec.LZ4
     assert barray.cparams.use_dict is False
 
+    barray = blosc2.BatchStore(cparams={"codec": blosc2.Codec.LZ4HC, "clevel": 1, "use_dict": True})
+    assert barray.cparams.codec == blosc2.Codec.LZ4HC
+    assert barray.cparams.use_dict is True
+
     barray = blosc2.BatchStore(cparams={"codec": blosc2.Codec.ZSTD, "clevel": 0})
     assert barray.cparams.codec == blosc2.Codec.ZSTD
     assert barray.cparams.use_dict is False
@@ -252,6 +256,27 @@ def test_batchstore_respects_explicit_use_dict_and_non_zstd():
     assert barray.cparams.use_dict is False
 
 
+def test_batchstore_guess_max_blocksize_uses_l1_for_low_clevel(monkeypatch):
+    monkeypatch.setitem(blosc2.cpu_info, "l1_data_cache_size", 100)
+    monkeypatch.setitem(blosc2.cpu_info, "l2_cache_size", 1000)
+    barray = blosc2.BatchStore(cparams={"clevel": 5})
+    assert barray._guess_blocksize([30, 30, 30, 30]) == 3
+
+
+def test_batchstore_guess_max_blocksize_uses_l2_for_mid_clevel(monkeypatch):
+    monkeypatch.setitem(blosc2.cpu_info, "l1_data_cache_size", 100)
+    monkeypatch.setitem(blosc2.cpu_info, "l2_cache_size", 150)
+    barray = blosc2.BatchStore(cparams={"clevel": 6})
+    assert barray._guess_blocksize([60, 60, 60, 60]) == 2
+
+
+def test_batchstore_guess_max_blocksize_uses_full_batch_for_clevel_9(monkeypatch):
+    monkeypatch.setitem(blosc2.cpu_info, "l1_data_cache_size", 1)
+    monkeypatch.setitem(blosc2.cpu_info, "l2_cache_size", 1)
+    barray = blosc2.BatchStore(cparams={"clevel": 9})
+    assert barray._guess_blocksize([100, 100, 100, 100]) == 4
+
+
 def test_vlcompress_small_blocks_roundtrip():
     values = [
         {"value": None},

tests/test_vlarray.py

@@ -155,6 +155,10 @@ def test_vlarray_respects_explicit_use_dict_and_non_zstd():
     assert vlarray.cparams.codec == blosc2.Codec.LZ4
     assert vlarray.cparams.use_dict is False
 
+    vlarray = blosc2.VLArray(cparams={"codec": blosc2.Codec.LZ4HC, "clevel": 1, "use_dict": True})
+    assert vlarray.cparams.codec == blosc2.Codec.LZ4HC
+    assert vlarray.cparams.use_dict is True
+
     vlarray = blosc2.VLArray(cparams={"codec": blosc2.Codec.ZSTD, "clevel": 0})
     assert vlarray.cparams.codec == blosc2.Codec.ZSTD
     assert vlarray.cparams.use_dict is False