cache_metadata

Author: alimanfoo

docs/release.rst

@@ -4,6 +4,7 @@ Release notes
 * Added ``overwrite`` keyword argument to array and group creation methods
   on the :class:`zarr.hierarchy.Group` class
   (`#71 <https://github.com/alimanfoo/zarr/issues/71>`_).
+* Added ``cache_metadata`` keyword argument to array creation methods.
 
 .. _release_2.0.1:
 

zarr/core.py

@@ -34,6 +34,11 @@ class Array(object):
         for storage of both chunks and metadata.
     synchronizer : object, optional
         Array synchronizer.
+    cache_metadata : bool, optional
+        If True, array configuration metadata will be cached for the
+        lifetime of the object. If False, array metadata will be reloaded
+        prior to all data access and modification operations (may incur
+        overhead depending on storage and data access pattern).
 
     Attributes
     ----------
@@ -71,7 +76,7 @@ class Array(object):
     """  # flake8: noqa
 
     def __init__(self, store, path=None, read_only=False, chunk_store=None,
-                 synchronizer=None):
+                 synchronizer=None, cache_metadata=True):
         # N.B., expect at this point store is fully initialized with all
         # configuration metadata fully specified and normalized
 
@@ -87,11 +92,21 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
         else:
             self._chunk_store = chunk_store
         self._synchronizer = synchronizer
+        self._cache_metadata = cache_metadata
+        self._is_view = False
 
         # initialize metadata
+        self._load_metadata()
+
+        # initialize attributes
+        akey = self._key_prefix + attrs_key
+        self._attrs = Attributes(store, key=akey, read_only=read_only,
+                                 synchronizer=synchronizer)
+
+    def _load_metadata(self):
         try:
             mkey = self._key_prefix + array_meta_key
-            meta_bytes = store[mkey]
+            meta_bytes = self._store[mkey]
         except KeyError:
             raise ValueError('store has no metadata')
         else:
@@ -104,7 +119,6 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
             self._dtype = meta['dtype']
             self._fill_value = meta['fill_value']
             self._order = meta['order']
-            self._is_view = False
 
             # setup compressor
             config = meta['compressor']
@@ -119,14 +133,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
                 filters = [get_codec(f) for f in filters]
             self._filters = filters
 
-        # initialize attributes
-        akey = self._key_prefix + attrs_key
-        self._attrs = Attributes(store, key=akey, read_only=read_only,
-                                 synchronizer=synchronizer)
-
     def _flush_metadata(self):
         if self._is_view:
             raise PermissionError('operation not permitted for views')
+
         if self._compressor:
             compressor_config = self._compressor.get_config()
         else:
@@ -366,6 +376,10 @@ def __getitem__(self, item):
 
         """  # flake8: noqa
 
+        # refresh metadata
+        if not self._cache_metadata:
+            self._load_metadata()
+
         # normalize selection
         selection = normalize_array_selection(item, self._shape)
 
@@ -484,6 +498,10 @@ def __setitem__(self, key, value):
         if self._read_only:
             raise PermissionError('array is read-only')
 
+        # refresh metadata
+        if not self._cache_metadata:
+            self._load_metadata()
+
         # normalize selection
         selection = normalize_array_selection(key, self._shape)
 
@@ -717,6 +735,10 @@ def _encode_chunk(self, chunk):
 
     def __repr__(self):
 
+        # refresh metadata
+        if not self._cache_metadata:
+            self._load_metadata()
+
         # main line
         r = '%s(' % type(self).__name__
         if self.name:
@@ -772,11 +794,24 @@ def _write_op(self, f, *args, **kwargs):
 
         # synchronization
         if self._synchronizer is None:
+
+            # refresh metadata
+            if not self._cache_metadata:
+                self._load_metadata()
+
             return f(*args, **kwargs)
+
         else:
+
             # synchronize on the array
             mkey = self._key_prefix + array_meta_key
+
             with self._synchronizer[mkey]:
+
+                # refresh metadata
+                if not self._cache_metadata:
+                    self._load_metadata()
+
                 return f(*args, **kwargs)
 
     def resize(self, *args):
@@ -1034,7 +1069,8 @@ def view(self, shape=None, chunks=None, dtype=None,
         if synchronizer is None:
             synchronizer = self._synchronizer
         a = Array(store=store, path=path, chunk_store=chunk_store,
-                  read_only=read_only, synchronizer=synchronizer)
+                  read_only=read_only, synchronizer=synchronizer,
+                  cache_metadata=True)
         a._is_view = True
 
         # allow override of some properties

zarr/creation.py

@@ -15,7 +15,7 @@
 def create(shape, chunks=None, dtype=None, compressor='default',
            fill_value=None, order='C', store=None, synchronizer=None,
            overwrite=False, path=None, chunk_store=None, filters=None,
-           **kwargs):
+           cache_metadata=True, **kwargs):
     """Create an array.
 
     Parameters
@@ -47,6 +47,11 @@ def create(shape, chunks=None, dtype=None, compressor='default',
         for storage of both chunks and metadata.
     filters : sequence of Codecs, optional
         Sequence of filters to use to encode chunk data prior to compression.
+    cache_metadata : bool, optional
+        If True, array configuration metadata will be cached for the
+        lifetime of the object. If False, array metadata will be reloaded
+        prior to all data access and modification operations (may incur
+        overhead depending on storage and data access pattern).
 
     Returns
     -------
@@ -82,7 +87,7 @@ def create(shape, chunks=None, dtype=None, compressor='default',
 
     # instantiate array
     z = Array(store, path=path, chunk_store=chunk_store,
-              synchronizer=synchronizer)
+              synchronizer=synchronizer, cache_metadata=cache_metadata)
 
     return z
 
@@ -277,7 +282,7 @@ def array(data, **kwargs):
 
 def open_array(path, mode='a', shape=None, chunks=None, dtype=None,
                compressor='default', fill_value=None, order='C',
-               synchronizer=None, filters=None, **kwargs):
+               synchronizer=None, filters=None, cache_metadata=True, **kwargs):
     """Convenience function to instantiate an array stored in a
     directory on the file system.
 
@@ -306,6 +311,11 @@ def open_array(path, mode='a', shape=None, chunks=None, dtype=None,
         Array synchronizer.
     filters : sequence, optional
         Sequence of filters to use to encode chunk data prior to compression.
+    cache_metadata : bool, optional
+        If True, array configuration metadata will be cached for the
+        lifetime of the object. If False, array metadata will be reloaded
+        prior to all data access and modification operations (may incur
+        overhead depending on storage and data access pattern).
 
     Returns
     -------
@@ -388,7 +398,8 @@ def open_array(path, mode='a', shape=None, chunks=None, dtype=None,
     read_only = mode == 'r'
 
     # instantiate array
-    z = Array(store, read_only=read_only, synchronizer=synchronizer)
+    z = Array(store, read_only=read_only, synchronizer=synchronizer,
+              cache_metadata=cache_metadata)
 
     return z
 

zarr/hierarchy.py

@@ -496,7 +496,7 @@ def require_groups(self, *names):
     def create_dataset(self, name, data=None, shape=None, chunks=None,
                        dtype=None, compressor='default', fill_value=None,
                        order='C', synchronizer=None, filters=None,
-                       overwrite=False, **kwargs):
+                       overwrite=False, cache_metadata=True, **kwargs):
         """Create an array.
 
         Parameters
@@ -525,6 +525,11 @@ def create_dataset(self, name, data=None, shape=None, chunks=None,
             compression.
         overwrite : bool, optional
             If True, replace any existing array or group with the given name.
+        cache_metadata : bool, optional
+            If True, array configuration metadata will be cached for the
+            lifetime of the object. If False, array metadata will be reloaded
+            prior to all data access and modification operations (may incur
+            overhead depending on storage and data access pattern).
 
         Returns
         -------
@@ -548,12 +553,14 @@ def create_dataset(self, name, data=None, shape=None, chunks=None,
                               shape=shape, chunks=chunks, dtype=dtype,
                               compressor=compressor, fill_value=fill_value,
                               order=order, synchronizer=synchronizer,
-                              filters=filters, overwrite=overwrite, **kwargs)
+                              filters=filters, overwrite=overwrite,
+                              cache_metadata=cache_metadata, **kwargs)
 
     def _create_dataset_nosync(self, name, data=None, shape=None, chunks=None,
                                dtype=None, compressor='default',
                                fill_value=None, order='C', synchronizer=None,
-                               filters=None, overwrite=False, **kwargs):
+                               filters=None, overwrite=False,
+                               cache_metadata=True, **kwargs):
 
         path = self._item_path(name)
 
@@ -568,15 +575,17 @@ def _create_dataset_nosync(self, name, data=None, shape=None, chunks=None,
                       order=order, synchronizer=synchronizer,
                       store=self._store, path=path,
                       chunk_store=self._chunk_store, filters=filters,
-                      overwrite=overwrite, **kwargs)
+                      overwrite=overwrite, cache_metadata=cache_metadata,
+                      **kwargs)
 
         else:
             a = create(shape=shape, chunks=chunks, dtype=dtype,
                        compressor=compressor, fill_value=fill_value,
                        order=order, synchronizer=synchronizer,
                        store=self._store, path=path,
                        chunk_store=self._chunk_store, filters=filters,
-                       overwrite=overwrite, **kwargs)
+                       overwrite=overwrite, cache_metadata=cache_metadata,
+                       **kwargs)
 
         return a
 
@@ -608,8 +617,10 @@ def _require_dataset_nosync(self, name, shape, dtype=None, exact=False,
 
         if contains_array(self._store, path):
             synchronizer = kwargs.get('synchronizer', self._synchronizer)
+            cache_metadata = kwargs.get('cache_metadata', True)
             a = Array(self._store, path=path, read_only=self._read_only,
-                      chunk_store=self._chunk_store, synchronizer=synchronizer)
+                      chunk_store=self._chunk_store,
+                      synchronizer=synchronizer, cache_metadata=cache_metadata)
             shape = normalize_shape(shape)
             if shape != a.shape:
                 raise TypeError('shapes do not match')

zarr/storage.py

@@ -123,7 +123,7 @@ def getsize(store, path=None):
 
 
 def _require_parent_group(path, store, chunk_store, overwrite):
-    path = normalize_storage_path(path)
+    # assume path is normalized
     if path:
         segments = path.split('/')
         for i in range(len(segments)):
@@ -249,6 +249,17 @@ def init_array(store, shape, chunks, dtype=None, compressor='default',
     _require_parent_group(path, store=store, chunk_store=chunk_store,
                           overwrite=overwrite)
 
+    _init_array_metadata(store, shape=shape, chunks=chunks, dtype=dtype,
+                         compressor=compressor, fill_value=fill_value,
+                         order=order, overwrite=overwrite, path=path,
+                         chunk_store=chunk_store, filters=filters)
+
+
+def _init_array_metadata(store, shape, chunks, dtype=None,
+                         compressor='default',
+                         fill_value=None, order='C', overwrite=False,
+                         path=None, chunk_store=None, filters=None):
+
     # guard conditions
     if overwrite:
         # attempt to delete any pre-existing items in store

zarr/tests/test_core.py

@@ -6,7 +6,6 @@
 import shutil
 import pickle
 from collections import MutableMapping
-import os
 
 
 import numpy as np
@@ -711,16 +710,13 @@ def test_repr(self):
 
 
 # custom store, does not support getsize()
-class CustomMapping(MutableMapping):
+class CustomMapping(object):
 
     def __init__(self):
         self.inner = dict()
 
-    def __iter__(self):
-        return iter(self.inner)
-
-    def __len__(self):
-        return len(self.inner)
+    def keys(self):
+        return self.inner.keys()
 
     def __getitem__(self, item):
         return self.inner[item]
@@ -764,3 +760,11 @@ def test_repr(self):
                 eq(l1, l2)
 
 
+class TestArrayNoCacheMetadata(TestArray):
+
+    @staticmethod
+    def create_array(read_only=False, **kwargs):
+        store = dict()
+        kwargs.setdefault('compressor', Zlib(level=1))
+        init_array(store, **kwargs)
+        return Array(store, read_only=read_only, cache_metadata=False)

zarr/tests/test_sync.py

@@ -78,7 +78,8 @@ def create_array(self, store=None, path=None, read_only=False,
         atexit.register(shutil.rmtree, sync_path)
         synchronizer = ProcessSynchronizer(sync_path)
         return Array(store, path=path, synchronizer=synchronizer,
-                     read_only=read_only, chunk_store=chunk_store)
+                     read_only=read_only, chunk_store=chunk_store,
+                     cache_metadata=False)
 
     def test_repr(self):
         if not PY2: