Add support for empty lists in VLArray store

Author: FrancescAlted

src/blosc2/_msgpack_utils.py

@@ -0,0 +1,26 @@
+#######################################################################
+# Copyright (c) 2019-present, Blosc Development Team <blosc@blosc.org>
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#######################################################################
+
+from __future__ import annotations
+
+from msgpack import packb, unpackb
+
+from blosc2 import blosc2_ext
+
+
+def msgpack_packb(value):
+    return packb(value, default=blosc2_ext.encode_tuple, strict_types=True, use_bin_type=True)
+
+
+def decode_tuple_list_hook(obj):
+    if obj and obj[0] == "__tuple__":
+        return tuple(obj[1:])
+    return obj
+
+
+def msgpack_unpackb(payload):
+    return unpackb(payload, list_hook=decode_tuple_list_hook)

src/blosc2/schunk.py

@@ -16,10 +16,10 @@
 from typing import Any, NamedTuple
 
 import numpy as np
-from msgpack import packb, unpackb
 
 import blosc2
 from blosc2 import SpecialValue, blosc2_ext
+from blosc2._msgpack_utils import msgpack_packb, msgpack_unpackb
 from blosc2.info import InfoReporter
 
 
@@ -46,12 +46,7 @@ def __setitem__(self, name, content):
                 return
             raise NotImplementedError("Slicing is not supported, unless [:]")
         cparams = {"typesize": 1}
-        content = packb(
-            content,
-            default=blosc2_ext.encode_tuple,
-            strict_types=True,
-            use_bin_type=True,
-        )
+        content = msgpack_packb(content)
         super().set_vlmeta(name, content, **cparams)
 
     def __getitem__(self, name):
@@ -60,7 +55,7 @@ def __getitem__(self, name):
                 # Return all the vlmetalayers
                 return self.getall()
             raise NotImplementedError("Slicing is not supported, unless [:]")
-        return unpackb(super().get_vlmeta(name), list_hook=blosc2_ext.decode_tuple)
+        return msgpack_unpackb(super().get_vlmeta(name))
 
     def __delitem__(self, name):
         blosc2_ext.check_access_mode(self.urlpath, self.mode)
@@ -120,7 +115,7 @@ def __setitem__(self, key: str, value: bytes) -> None:
             ..warning: Note that the *length* of the metalayer cannot change,
             otherwise an exception will be raised.
         """
-        value = packb(value, default=blosc2_ext.encode_tuple, strict_types=True, use_bin_type=True)
+        value = msgpack_packb(value)
         blosc2_ext.meta__setitem__(self.schunk, key, value)
 
     def __getitem__(self, item: str | slice) -> bytes | dict[str, bytes]:
@@ -144,10 +139,7 @@ def __getitem__(self, item: str | slice) -> bytes | dict[str, bytes]:
                 return self.getall()
             raise NotImplementedError("Slicing is not supported, unless [:]")
         if self.__contains__(item):
-            return unpackb(
-                blosc2_ext.meta__getitem__(self.schunk, item),
-                list_hook=blosc2_ext.decode_tuple,
-            )
+            return msgpack_unpackb(blosc2_ext.meta__getitem__(self.schunk, item))
         else:
             raise KeyError(f"{item} not found")
 

src/blosc2/vlarray.py

@@ -11,10 +11,8 @@
 import pathlib
 from typing import TYPE_CHECKING, Any
 
-from msgpack import packb, unpackb
-
 import blosc2
-from blosc2 import blosc2_ext
+from blosc2._msgpack_utils import msgpack_packb, msgpack_unpackb
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -177,7 +175,7 @@ def _copy_meta(self) -> dict[str, Any]:
         return {name: self.meta[name] for name in self.meta}
 
     def _serialize(self, value: Any) -> bytes:
-        payload = packb(value, default=blosc2_ext.encode_tuple, strict_types=True, use_bin_type=True)
+        payload = msgpack_packb(value)
         _check_serialized_size(payload)
         return payload
 
@@ -244,7 +242,7 @@ def __getitem__(self, index: int) -> Any:
             return [self[i] for i in self._slice_indices(index)]
         index = self._normalize_index(index)
         payload = self.schunk.decompress_chunk(index)
-        return unpackb(payload, list_hook=blosc2_ext.decode_tuple)
+        return msgpack_unpackb(payload)
 
     def __setitem__(self, index: int, value: Any) -> None:
         if isinstance(index, slice):
@@ -335,7 +333,7 @@ def __repr__(self) -> str:
         return f"VLArray(len={len(self)}, urlpath={self.urlpath!r})"
 
 
-def vlarray_from_cframe(cframe: bytes, copy: bool = False) -> VLArray:
+def vlarray_from_cframe(cframe: bytes, copy: bool = True) -> VLArray:
     """Deserialize a CFrame buffer into a :class:`VLArray`."""
 
     schunk = blosc2.schunk_from_cframe(cframe, copy=copy)

tests/test_vlarray.py

@@ -255,6 +255,13 @@ def test_vlarray_copy():
     blosc2.remove_urlpath(copy_path)
 
 
+def test_vlarray_empty_list_roundtrip():
+    values = [[], {"a": []}, [[], ["nested"]], None, ("tuple", []), {"rows": [[], []]}]
+    vlarray = blosc2.VLArray()
+    vlarray.extend(values)
+    assert list(vlarray) == values
+
+
 def test_vlarray_insert_delete_errors():
     vlarray = blosc2.VLArray()
     vlarray.append("value")

tests/test_vlmeta.py

@@ -118,3 +118,12 @@ def clear(schunk):
 
     schunk.vlmeta.clear()
     assert schunk.vlmeta.__len__() == 0
+
+
+def test_vlmeta_empty_list_roundtrip():
+    schunk = blosc2.SChunk()
+    schunk.vlmeta["empty"] = []
+    schunk.vlmeta["nested"] = {"rows": [[], ["x"]]}
+
+    assert schunk.vlmeta["empty"] == []
+    assert schunk.vlmeta["nested"] == {"rows": [[], ["x"]]}