Add several list-oriented methods (insert, delete, pop...); docs are here too

Author: FrancescAlted

doc/reference/classes.rst

@@ -16,6 +16,7 @@ Main Classes
     DictStore
     TreeStore
     EmbedStore
+    VLArray
     Proxy
     ProxySource
     ProxyNDSource
@@ -33,6 +34,7 @@ Main Classes
     dict_store
     tree_store
     embed_store
+    vlarray
     proxy
     proxysource
     proxyndsource

doc/reference/misc.rst

@@ -134,6 +134,8 @@ This page documents the miscellaneous members of the ``blosc2`` module that do n
         TreeStore,
         DictStore,
         EmbedStore,
+        VLArray,
+        vlarray_from_cframe,
         abs,
         acos,
         acosh,

src/blosc2/vlarray.py

@@ -89,6 +89,16 @@ def _maybe_open_existing(self, storage: blosc2.Storage) -> bool:
         self._attach_schunk(schunk)
         return True
 
+    def _make_storage(self) -> blosc2.Storage:
+        meta = {name: self.meta[name] for name in self.meta}
+        return blosc2.Storage(
+            contiguous=self.schunk.contiguous,
+            urlpath=self.urlpath,
+            mode=self.mode,
+            mmap_mode=self.mmap_mode,
+            meta=meta,
+        )
+
     def __init__(
         self,
         chunksize: int | None = None,
@@ -149,6 +159,17 @@ def _normalize_index(self, index: int) -> int:
             raise IndexError("VLArray index out of range")
         return index
 
+    def _normalize_insert_index(self, index: int) -> int:
+        if not isinstance(index, int):
+            raise TypeError("VLArray indices must be integers")
+        if index < 0:
+            index += len(self)
+            if index < 0:
+                return 0
+        if index > len(self):
+            return len(self)
+        return index
+
     def _serialize(self, value: Any) -> bytes:
         payload = packb(value, default=blosc2_ext.encode_tuple, strict_types=True, use_bin_type=True)
         _check_serialized_size(payload)
@@ -158,10 +179,58 @@ def _compress(self, payload: bytes) -> bytes:
         return blosc2.compress2(payload, cparams=self.schunk.cparams)
 
     def append(self, value: Any) -> int:
+        """Append one value and return the new number of entries."""
         self._check_writable()
         chunk = self._compress(self._serialize(value))
         return self.schunk.append_chunk(chunk)
 
+    def insert(self, index: int, value: Any) -> int:
+        """Insert one value at ``index`` and return the new number of entries."""
+        self._check_writable()
+        index = self._normalize_insert_index(index)
+        chunk = self._compress(self._serialize(value))
+        return self.schunk.insert_chunk(index, chunk)
+
+    def delete(self, index: int) -> int:
+        """Delete the value at ``index`` and return the new number of entries."""
+        self._check_writable()
+        if isinstance(index, slice):
+            raise NotImplementedError("Slicing is not supported for VLArray")
+        index = self._normalize_index(index)
+        return self.schunk.delete_chunk(index)
+
+    def pop(self, index: int = -1) -> Any:
+        """Remove and return the value at ``index``."""
+        self._check_writable()
+        if isinstance(index, slice):
+            raise NotImplementedError("Slicing is not supported for VLArray")
+        index = self._normalize_index(index)
+        value = self[index]
+        self.schunk.delete_chunk(index)
+        return value
+
+    def extend(self, values: object) -> None:
+        """Append all values from an iterable."""
+        self._check_writable()
+        for value in values:
+            chunk = self._compress(self._serialize(value))
+            self.schunk.append_chunk(chunk)
+
+    def clear(self) -> None:
+        """Remove all entries from the container."""
+        self._check_writable()
+        storage = self._make_storage()
+        if storage.urlpath is not None:
+            blosc2.remove_urlpath(storage.urlpath)
+        schunk = blosc2.SChunk(
+            chunksize=-1,
+            data=None,
+            cparams=copy.deepcopy(self.cparams),
+            dparams=copy.deepcopy(self.dparams),
+            storage=storage,
+        )
+        self._attach_schunk(schunk)
+
     def __getitem__(self, index: int) -> Any:
         if isinstance(index, slice):
             raise NotImplementedError("Slicing is not supported for VLArray")
@@ -177,6 +246,9 @@ def __setitem__(self, index: int, value: Any) -> None:
         chunk = self._compress(self._serialize(value))
         self.schunk.update_chunk(index, chunk)
 
+    def __delitem__(self, index: int) -> None:
+        self.delete(index)
+
     def __len__(self) -> int:
         return self.schunk.nchunks
 

tests/test_vlarray.py

@@ -53,6 +53,16 @@ def test_vlarray_roundtrip(contiguous, urlpath):
     expected[-1] = "tiny"
     vlarray[1] = expected[1]
     vlarray[-1] = expected[-1]
+    assert vlarray.insert(0, "head") == len(expected) + 1
+    expected.insert(0, "head")
+    assert vlarray.insert(-1, {"between": 5}) == len(expected) + 1
+    expected.insert(-1, {"between": 5})
+    assert vlarray.insert(999, "tail") == len(expected) + 1
+    expected.insert(999, "tail")
+    assert vlarray.delete(2) == len(expected) - 1
+    del expected[2]
+    del vlarray[-2]
+    del expected[-2]
     assert list(vlarray) == expected
 
     if urlpath is not None:
@@ -63,6 +73,18 @@ def test_vlarray_roundtrip(contiguous, urlpath):
             reopened.append("nope")
         with pytest.raises(ValueError):
             reopened[0] = "nope"
+        with pytest.raises(ValueError):
+            reopened.insert(0, "nope")
+        with pytest.raises(ValueError):
+            reopened.delete(0)
+        with pytest.raises(ValueError):
+            del reopened[0]
+        with pytest.raises(ValueError):
+            reopened.extend(["nope"])
+        with pytest.raises(ValueError):
+            reopened.pop()
+        with pytest.raises(ValueError):
+            reopened.clear()
 
         reopened_rw = blosc2.open(urlpath, mode="a")
         reopened_rw[0] = "changed"
@@ -79,28 +101,32 @@ def test_vlarray_roundtrip(contiguous, urlpath):
 
 def test_vlarray_from_cframe():
     vlarray = blosc2.VLArray()
-    for value in VALUES[:4]:
-        vlarray.append(value)
+    vlarray.extend(VALUES)
+    vlarray.insert(1, {"inserted": True})
+    del vlarray[3]
+    expected = list(VALUES)
+    expected.insert(1, {"inserted": True})
+    del expected[3]
 
     restored = blosc2.from_cframe(vlarray.to_cframe())
     assert isinstance(restored, blosc2.VLArray)
-    assert list(restored) == VALUES[:4]
+    assert list(restored) == expected
 
     restored2 = blosc2.vlarray_from_cframe(vlarray.to_cframe())
     assert isinstance(restored2, blosc2.VLArray)
-    assert list(restored2) == VALUES[:4]
+    assert list(restored2) == expected
 
 
 def test_vlarray_constructor_kwargs():
     urlpath = "test_vlarray_kwargs.b2frame"
     blosc2.remove_urlpath(urlpath)
 
     vlarray = blosc2.VLArray(urlpath=urlpath, mode="w", contiguous=True)
-    for value in VALUES[:3]:
+    for value in VALUES:
         vlarray.append(value)
 
     reopened = blosc2.VLArray(urlpath=urlpath, mode="r", contiguous=True, mmap_mode="r")
-    assert list(reopened) == VALUES[:3]
+    assert list(reopened) == VALUES
 
     blosc2.remove_urlpath(urlpath)
 
@@ -110,3 +136,52 @@ def test_vlarray_size_guard(monkeypatch):
     monkeypatch.setattr(blosc2, "MAX_BUFFERSIZE", 4)
     with pytest.raises(ValueError, match="Serialized objects cannot be larger"):
         vlarray.append("payload")
+
+
+@pytest.mark.parametrize(
+    ("contiguous", "urlpath"),
+    [
+        (False, None),
+        (True, None),
+        (True, "test_vlarray_list_ops.b2frame"),
+        (False, "test_vlarray_list_ops_s.b2frame"),
+    ],
+)
+def test_vlarray_list_like_ops(contiguous, urlpath):
+    blosc2.remove_urlpath(urlpath)
+
+    vlarray = blosc2.VLArray(storage=_storage(contiguous, urlpath))
+    vlarray.extend([1, 2, 3])
+    assert list(vlarray) == [1, 2, 3]
+    assert vlarray.pop() == 3
+    assert vlarray.pop(0) == 1
+    assert list(vlarray) == [2]
+
+    vlarray.clear()
+    assert len(vlarray) == 0
+    assert list(vlarray) == []
+
+    vlarray.extend(["a", "b"])
+    assert list(vlarray) == ["a", "b"]
+
+    if urlpath is not None:
+        reopened = blosc2.open(urlpath, mode="r")
+        assert list(reopened) == ["a", "b"]
+
+    blosc2.remove_urlpath(urlpath)
+
+
+def test_vlarray_insert_delete_errors():
+    vlarray = blosc2.VLArray()
+    vlarray.append("value")
+
+    with pytest.raises(TypeError):
+        vlarray.insert("0", "bad")
+    with pytest.raises(IndexError):
+        vlarray.delete(3)
+    with pytest.raises(NotImplementedError):
+        vlarray.delete(slice(0, 1))
+    with pytest.raises(IndexError):
+        blosc2.VLArray().pop()
+    with pytest.raises(NotImplementedError):
+        vlarray.pop(slice(0, 1))