Add new (and persistent) meta layers for DictStore and TreeStore. Fixes #546.

Author: FrancescAlted

src/blosc2/dict_store.py

@@ -23,6 +23,7 @@
 class DictStore:
     """
     Directory-based storage for compressed data using Blosc2.
+
     Manages arrays in a directory (.b2d) or zip (.b2z) format.
 
     Supports the following types:
@@ -102,6 +103,7 @@ def __init__(
         threshold: int | None = 2**13,
         *,
         mmap_mode: str | None = None,
+        _storage_meta: dict | None = None,
     ):
         """
         See :class:`DictStore` for full documentation of parameters.
@@ -123,6 +125,12 @@ def __init__(
         self.dparams = dparams or blosc2.DParams()
         self.storage = storage or blosc2.Storage()
 
+        if _storage_meta:
+            self.storage.meta = _storage_meta
+        else:
+            # Mark this storage as a b2dict object
+            self.storage.meta = {"b2dict": {"version": 1}}
+
         self.offsets = {}
         self.map_tree = {}
         self._temp_dir_obj = None
@@ -187,6 +195,7 @@ def _init_read_mode(self, dparams: blosc2.DParams | None = None):
             self._update_map_tree()
 
         self._estore = EmbedStore(_from_schunk=schunk)
+        self.storage.meta = self._estore.storage.meta
 
     def _init_write_append_mode(
         self,
@@ -208,6 +217,7 @@ def _init_write_append_mode(
             cparams=cparams,
             dparams=dparams,
             storage=storage,
+            meta=self.storage.meta,
         )
         self._update_map_tree()
 

src/blosc2/embed_store.py

@@ -81,6 +81,7 @@ def __init__(
         _from_schunk: SChunk | None = None,
         *,
         mmap_mode: str | None = None,
+        meta: dict | None = None,
     ):
         """Initialize EmbedStore."""
 
@@ -100,6 +101,8 @@ def __init__(
             self.dparams = _from_schunk.dparams
             self.mode = mode
             self._store = _from_schunk
+            self.storage = blosc2.Storage()
+            self.storage.meta = _from_schunk.meta
             self._load_metadata()
             return
 
@@ -119,14 +122,14 @@ def __init__(
 
         if mode in ("r", "a") and urlpath:
             self._store = blosc2.blosc2_ext.open(urlpath, mode=mode, offset=0, mmap_mode=mmap_mode)
+            self.storage.meta = self._store.meta
             self._load_metadata()
             return
 
         _cparams = copy.deepcopy(self.cparams)
         _cparams.typesize = 1  # ensure typesize is set to 1 for byte storage
         _storage = self.storage
-        # Mark this storage as a b2embed object
-        _storage.meta = {"b2embed": {"version": 1}}
+        _storage.meta = meta if meta is not None else {"b2embed": {"version": 1}}
         if self._schunk_store:
             self._store = blosc2.SChunk(
                 chunksize=chunksize,

src/blosc2/tree_store.py

@@ -149,7 +149,7 @@ class TreeStore(DictStore):
     """
 
     # For some reason, we had to revert the explicit parametrisation of the
-    # constructor to make benchmarks wrok fine again.
+    # constructor to make benchmarks working again.
     def __init__(self, *args, _from_parent_store=None, **kwargs):
         """Initialize TreeStore with subtree support.
 
@@ -159,7 +159,9 @@ def __init__(self, *args, _from_parent_store=None, **kwargs):
             # This is a subtree view, copy state from parent
             self.__dict__.update(_from_parent_store.__dict__)
         else:
-            super().__init__(*args, **kwargs)
+            # Call initialization and mark this storage as a b2tree object
+            super().__init__(*args, **kwargs, _storage_meta={"b2tree": {"version": 1}})
+
             self.subtree_path = ""  # Empty string means full tree
 
     def _is_vlmeta_key(self, key: str) -> bool:

tests/test_dict_store.py

@@ -48,6 +48,7 @@ def populated_dict_store(request):
 
 def test_basic_dstore(populated_dict_store):
     dstore, path = populated_dict_store
+    assert "b2dict" in dstore.storage.meta
     assert set(dstore.keys()) == {"/node1", "/node2", "/dir1/node3"}
     assert np.all(dstore["/node1"][:] == np.array([1, 2, 3]))
     assert np.all(dstore["/node2"][:] == np.ones(2))
@@ -64,6 +65,7 @@ def test_basic_dstore(populated_dict_store):
     # Persist and reopen
     dstore.close()
     with DictStore(path, mode="r") as dstore_read:
+        assert "b2dict" in dstore_read.storage.meta
         keys = set(dstore_read.keys())
         assert "/node2" in keys
         assert "/dir1/node3" in keys

tests/test_embed_store.py

@@ -42,6 +42,7 @@ def populate_nodes(cleanup_files):
 def test_basic(populate_nodes):
     estore = populate_nodes
 
+    assert "b2embed" in estore.storage.meta
     assert set(estore.keys()) == {"/node1", "/node2", "/node3"}
     assert np.all(estore["/node1"][:] == np.array([1, 2, 3]))
     assert np.all(estore["/node2"][:] == np.arange(3))
@@ -51,6 +52,7 @@ def test_basic(populate_nodes):
     assert "/node1" not in estore
 
     estore_read = blosc2.EmbedStore(urlpath="test_estore.b2e", mode="r")
+    assert "b2embed" in estore_read.storage.meta
     assert set(estore_read.keys()) == {"/node2", "/node3"}
     for value in estore_read.values():
         assert hasattr(value, "shape")

tests/test_tree_store.py

@@ -49,6 +49,8 @@ def test_basic_tree_store(populated_tree_store):
     """Test basic TreeStore functionality."""
     tstore, _ = populated_tree_store
 
+    assert "b2tree" in tstore.storage.meta
+
     # Test key existence - should include both leaf and structural nodes
     expected_keys = {
         "/child0/data",
@@ -450,6 +452,7 @@ def test_treestore_vlmeta_basic_and_bulk(storage_type):
 
     # Reopen in read-only to check persistence and read-only protection
     with TreeStore(path, mode="r") as tstore:
+        assert "b2tree" in tstore.storage.meta
         assert tstore.vlmeta["author"] == "blosc2"
         assert tstore.vlmeta["version"] == 1
         assert tstore.vlmeta["shape"] == (3, 2)