Merge pull request #220 from alimanfoo/cache-attrs-20171224b

Cache attributes

Author: alimanfoo

docs/api.rst

@@ -8,6 +8,7 @@ API reference
     api/core
     api/hierarchy
     api/storage
+    api/convenience
     api/codecs
+    api/attrs
     api/sync
-    api/convenience

docs/api/attrs.rst

@@ -0,0 +1,16 @@
+The Attributes class (``zarr.attrs``)
+=====================================
+.. module:: zarr.attrs
+
+.. autoclass:: Attributes
+
+    .. automethod:: __getitem__
+    .. automethod:: __setitem__
+    .. automethod:: __delitem__
+    .. automethod:: __iter__
+    .. automethod:: __len__
+    .. automethod:: keys
+    .. automethod:: asdict
+    .. automethod:: put
+    .. automethod:: update
+    .. automethod:: refresh

docs/release.rst

@@ -127,6 +127,10 @@ Enhancements
 * **Added support for ``datetime64`` and ``timedelta64`` data types**;
   :issue:`85`, :issue:`215`.
 
+* **Array and group attributes are now cached by default** to improve performance with
+  slow stores, e.g., stores accessing data via the network; :issue:`220`, :issue:`218`,
+  :issue:`204`.
+
 * **New LRUStoreCache class**. The class :class:`zarr.storage.LRUStoreCache` has been
   added and provides a means to locally cache data in memory from a store that may be
   slow, e.g., a store that retrieves data from a remote server via the network;

zarr/attrs.py

@@ -9,24 +9,63 @@
 
 
 class Attributes(MutableMapping):
-
-    def __init__(self, store, key='.zattrs', read_only=False,
+    """Class providing access to user attributes on an array or group. Should not be
+    instantiated directly, will be available via the `.attrs` property of an array or
+    group.
+
+    Parameters
+    ----------
+    store : MutableMapping
+        The store in which to store the attributes.
+    key : str, optional
+        The key under which the attributes will be stored.
+    read_only : bool, optional
+        If True, attributes cannot be modified.
+    cache : bool, optional
+        If True (default), attributes will be cached locally.
+    synchronizer : Synchronizer
+        Only necessary if attributes may be modified from multiple threads or processes.
+
+    """
+
+    def __init__(self, store, key='.zattrs', read_only=False, cache=True,
                  synchronizer=None):
         self.store = store
         self.key = key
         self.read_only = read_only
+        self.cache = cache
+        self._cached_asdict = None
         self.synchronizer = synchronizer
 
+    def _get_nosync(self):
+        try:
+            data = self.store[self.key]
+        except KeyError:
+            d = dict()
+        else:
+            d = json.loads(text_type(data, 'ascii'))
+        return d
+
+    def asdict(self):
+        """Retrieve all attributes as a dictionary."""
+        if self.cache and self._cached_asdict is not None:
+            return self._cached_asdict
+        d = self._get_nosync()
+        if self.cache:
+            self._cached_asdict = d
+        return d
+
+    def refresh(self):
+        """Refresh cached attributes from the store."""
+        if self.cache:
+            self._cached_asdict = self._get_nosync()
+
     def __contains__(self, x):
         return x in self.asdict()
 
     def __getitem__(self, item):
         return self.asdict()[item]
 
-    def _put(self, d):
-        s = json.dumps(d, indent=4, sort_keys=True, ensure_ascii=True, separators=(',', ': '))
-        self.store[self.key] = s.encode('ascii')
-
     def _write_op(self, f, *args, **kwargs):
 
         # guard condition
@@ -46,48 +85,56 @@ def __setitem__(self, item, value):
     def _setitem_nosync(self, item, value):
 
         # load existing data
-        d = self.asdict()
+        d = self._get_nosync()
 
         # set key value
         d[item] = value
 
         # _put modified data
-        self._put(d)
+        self._put_nosync(d)
 
     def __delitem__(self, item):
         self._write_op(self._delitem_nosync, item)
 
     def _delitem_nosync(self, key):
 
         # load existing data
-        d = self.asdict()
+        d = self._get_nosync()
 
         # delete key value
         del d[key]
 
         # _put modified data
-        self._put(d)
+        self._put_nosync(d)
 
-    def asdict(self):
-        if self.key in self.store:
-            return json.loads(text_type(self.store[self.key], 'ascii'))
-        else:
-            return dict()
+    def put(self, d):
+        """Overwrite all attributes with the key/value pairs in the provided dictionary
+        `d` in a single operation."""
+        self._write_op(self._put_nosync, d)
+
+    def _put_nosync(self, d):
+        s = json.dumps(d, indent=4, sort_keys=True, ensure_ascii=True, separators=(',', ': '))
+        self.store[self.key] = s.encode('ascii')
+        if self.cache:
+            self._cached_asdict = d
 
     def update(self, *args, **kwargs):
-        # override to provide update in a single write
+        """Update the values of several attributes in a single operation."""
         self._write_op(self._update_nosync, *args, **kwargs)
 
     def _update_nosync(self, *args, **kwargs):
 
         # load existing data
-        d = self.asdict()
+        d = self._get_nosync()
 
         # update
         d.update(*args, **kwargs)
 
         # _put modified data
-        self._put(d)
+        self._put_nosync(d)
+
+    def keys(self):
+        return self.asdict().keys()
 
     def __iter__(self):
         return iter(self.asdict())

zarr/core.py

@@ -43,10 +43,14 @@ class Array(object):
     synchronizer : object, optional
         Array synchronizer.
     cache_metadata : bool, optional
-        If True, array configuration metadata will be cached for the
+        If True (default), array configuration metadata will be cached for the
         lifetime of the object. If False, array metadata will be reloaded
         prior to all data access and modification operations (may incur
         overhead depending on storage and data access pattern).
+    cache_attrs : bool, optional
+        If True (default), user attributes will be cached for attribute read
+        operations. If False, user attributes are reloaded from the store prior
+        to all attribute read operations.
 
     Attributes
     ----------
@@ -99,7 +103,7 @@ class Array(object):
     """
 
     def __init__(self, store, path=None, read_only=False, chunk_store=None,
-                 synchronizer=None, cache_metadata=True):
+                 synchronizer=None, cache_metadata=True, cache_attrs=True):
         # N.B., expect at this point store is fully initialized with all
         # configuration metadata fully specified and normalized
 
@@ -121,7 +125,7 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
         # initialize attributes
         akey = self._key_prefix + attrs_key
         self._attrs = Attributes(store, key=akey, read_only=read_only,
-                                 synchronizer=synchronizer)
+                                 synchronizer=synchronizer, cache=cache_attrs)
 
         # initialize info reporter
         self._info_reporter = InfoReporter(self)
@@ -1920,7 +1924,7 @@ def hexdigest(self, hashname="sha1"):
 
     def __getstate__(self):
         return (self._store, self._path, self._read_only, self._chunk_store,
-                self._synchronizer, self._cache_metadata)
+                self._synchronizer, self._cache_metadata, self._attrs.cache)
 
     def __setstate__(self, state):
         self.__init__(*state)

zarr/creation.py

@@ -16,8 +16,8 @@
 def create(shape, chunks=True, dtype=None, compressor='default',
            fill_value=0, order='C', store=None, synchronizer=None,
            overwrite=False, path=None, chunk_store=None, filters=None,
-           cache_metadata=True, read_only=False, object_codec=None,
-           **kwargs):
+           cache_metadata=True, cache_attrs=True, read_only=False,
+           object_codec=None, **kwargs):
     """Create an array.
 
     Parameters
@@ -54,6 +54,10 @@ def create(shape, chunks=True, dtype=None, compressor='default',
         lifetime of the object. If False, array metadata will be reloaded
         prior to all data access and modification operations (may incur
         overhead depending on storage and data access pattern).
+    cache_attrs : bool, optional
+        If True (default), user attributes will be cached for attribute read
+        operations. If False, user attributes are reloaded from the store prior
+        to all attribute read operations.
     read_only : bool, optional
         True if array should be protected against modification.
     object_codec : Codec, optional
@@ -115,7 +119,7 @@ def create(shape, chunks=True, dtype=None, compressor='default',
 
     # instantiate array
     z = Array(store, path=path, chunk_store=chunk_store, synchronizer=synchronizer,
-              cache_metadata=cache_metadata, read_only=read_only)
+              cache_metadata=cache_metadata, cache_attrs=cache_attrs, read_only=read_only)
 
     return z
 
@@ -342,8 +346,9 @@ def array(data, **kwargs):
 
 
 def open_array(store, mode='a', shape=None, chunks=True, dtype=None, compressor='default',
-               fill_value=0, order='C', synchronizer=None, filters=None, cache_metadata=True,
-               path=None, object_codec=None, **kwargs):
+               fill_value=0, order='C', synchronizer=None, filters=None,
+               cache_metadata=True, cache_attrs=True, path=None, object_codec=None,
+               **kwargs):
     """Open an array using file-mode-like semantics.
 
     Parameters
@@ -377,6 +382,10 @@ def open_array(store, mode='a', shape=None, chunks=True, dtype=None, compressor=
         lifetime of the object. If False, array metadata will be reloaded
         prior to all data access and modification operations (may incur
         overhead depending on storage and data access pattern).
+    cache_attrs : bool, optional
+        If True (default), user attributes will be cached for attribute read
+        operations. If False, user attributes are reloaded from the store prior
+        to all attribute read operations.
     path : string, optional
         Array path within store.
     object_codec : Codec, optional
@@ -465,7 +474,7 @@ def open_array(store, mode='a', shape=None, chunks=True, dtype=None, compressor=
 
     # instantiate array
     z = Array(store, read_only=read_only, synchronizer=synchronizer,
-              cache_metadata=cache_metadata, path=path)
+              cache_metadata=cache_metadata, cache_attrs=cache_attrs, path=path)
 
     return z