open... functions accept any store; resolves #56

Author: alimanfoo

docs/release.rst

@@ -5,6 +5,9 @@ Release notes
   on the :class:`zarr.hierarchy.Group` class
   (`#71 <https://github.com/alimanfoo/zarr/issues/71>`_).
 * Added ``cache_metadata`` keyword argument to array creation methods.
+* The functions :func:`zarr.creation.open_array` and
+  :func:`zarr.hierarchy.open_group` now accept any store as first argument
+  (`#56 <https://github.com/alimanfoo/zarr/issues/56>`_).
 
 .. _release_2.0.1:
 

zarr/creation.py

@@ -32,9 +32,8 @@ def create(shape, chunks=None, dtype=None, compressor='default',
         Default value to use for uninitialized portions of the array.
     order : {'C', 'F'}, optional
         Memory layout to be used within each chunk.
-    store : MutableMapping, optional
-        Array storage. If not provided, a Python dict will be used, meaning
-        array data will be stored in memory.
+    store : MutableMapping or string
+        Store or path to directory in file system.
     synchronizer : object, optional
         Array synchronizer.
     overwrite : bool, optional
@@ -72,9 +71,8 @@ def create(shape, chunks=None, dtype=None, compressor='default',
 
     """  # flake8: noqa
 
-    # initialize store
-    if store is None:
-        store = dict()
+    # handle polymorphic store arg
+    store = _handle_store_arg(store)
 
     # compatibility
     compressor, fill_value = _handle_kwargs(compressor, fill_value, kwargs)
@@ -92,6 +90,15 @@ def create(shape, chunks=None, dtype=None, compressor='default',
     return z
 
 
+def _handle_store_arg(store):
+    if store is None:
+        return dict()
+    elif isinstance(store, str):
+        return DirectoryStore(store)
+    else:
+        return store
+
+
 def _handle_kwargs(compressor, fill_value, kwargs):
 
     # to be compatible with h5py, as well as backwards-compatible with Zarr
@@ -280,16 +287,16 @@ def array(data, **kwargs):
     return z
 
 
-def open_array(path, mode='a', shape=None, chunks=None, dtype=None,
+def open_array(store=None, mode='a', shape=None, chunks=None, dtype=None,
                compressor='default', fill_value=None, order='C',
-               synchronizer=None, filters=None, cache_metadata=True, **kwargs):
-    """Convenience function to instantiate an array stored in a
-    directory on the file system.
+               synchronizer=None, filters=None, cache_metadata=True,
+               path=None, **kwargs):
+    """Open array using mode-like semantics.
 
     Parameters
     ----------
-    path : string
-        Path to directory in file system in which to store the array.
+    store : MutableMapping or string
+        Store or path to directory in file system.
     mode : {'r', 'r+', 'a', 'w', 'w-'}
         Persistence mode: 'r' means read only (must exist); 'r+' means
         read/write (must exist); 'a' means read/write (create if doesn't
@@ -316,6 +323,8 @@ def open_array(path, mode='a', shape=None, chunks=None, dtype=None,
         lifetime of the object. If False, array metadata will be reloaded
         prior to all data access and modification operations (may incur
         overhead depending on storage and data access pattern).
+    path : string, optional
+        Array path.
 
     Returns
     -------
@@ -349,57 +358,56 @@ def open_array(path, mode='a', shape=None, chunks=None, dtype=None,
 
     """  # flake8: noqa
 
-    # use same mode semantics as h5py, although N.B., here `path` is a
-    # directory:
+    # use same mode semantics as h5py
     # r : read only, must exist
     # r+ : read/write, must exist
     # w : create, delete if exists
     # w- or x : create, fail if exists
     # a : read/write if exists, create otherwise (default)
 
-    # setup store
-    store = DirectoryStore(path)
+    # handle polymorphic store arg
+    store = _handle_store_arg(store)
 
     # compatibility
     compressor, fill_value = _handle_kwargs(compressor, fill_value, kwargs)
 
     # ensure store is initialized
 
     if mode in ['r', 'r+']:
-        if contains_group(store):
+        if contains_group(store, path=path):
             raise ValueError('store contains group')
-        elif not contains_array(store):
+        elif not contains_array(store, path=path):
             raise ValueError('array does not exist')
 
     elif mode == 'w':
         init_array(store, shape=shape, chunks=chunks, dtype=dtype,
                    compressor=compressor, fill_value=fill_value,
-                   order=order, filters=filters, overwrite=True)
+                   order=order, filters=filters, overwrite=True, path=path)
 
     elif mode == 'a':
-        if contains_group(store):
+        if contains_group(store, path=path):
             raise ValueError('store contains group')
-        elif not contains_array(store):
+        elif not contains_array(store, path=path):
             init_array(store, shape=shape, chunks=chunks, dtype=dtype,
                        compressor=compressor, fill_value=fill_value,
-                       order=order, filters=filters)
+                       order=order, filters=filters, path=path)
 
     elif mode in ['w-', 'x']:
-        if contains_group(store):
+        if contains_group(store, path=path):
             raise ValueError('store contains group')
-        elif contains_array(store):
+        elif contains_array(store, path=path):
             raise ValueError('store contains array')
         else:
             init_array(store, shape=shape, chunks=chunks, dtype=dtype,
                        compressor=compressor, fill_value=fill_value,
-                       order=order, filters=filters)
+                       order=order, filters=filters, path=path)
 
     # determine read only status
     read_only = mode == 'r'
 
     # instantiate array
     z = Array(store, read_only=read_only, synchronizer=synchronizer,
-              cache_metadata=cache_metadata)
+              cache_metadata=cache_metadata, path=path)
 
     return z
 

zarr/hierarchy.py

@@ -22,10 +22,10 @@ class Group(Mapping):
 
     Parameters
     ----------
-    store : HierarchicalStore
+    store : MutableMapping
         Group store, already initialized.
     path : string, optional
-        Storage path.
+        Group path.
     read_only : bool, optional
         True if group should be protected against modification.
     chunk_store : MutableMapping, optional
@@ -749,14 +749,23 @@ def _full_like_nosync(self, name, data, **kwargs):
                          chunk_store=self._chunk_store, **kwargs)
 
 
-def group(store=None, overwrite=False, chunk_store=None, synchronizer=None):
+def _handle_store_arg(store):
+    if store is None:
+        return DictStore()
+    elif isinstance(store, str):
+        return DirectoryStore(store)
+    else:
+        return store
+
+
+def group(store=None, overwrite=False, chunk_store=None, synchronizer=None,
+          path=None):
     """Create a group.
 
     Parameters
     ----------
-    store : MutableMapping, optional
-        Group storage. If not provided, a DictStore will be used, meaning
-        that data will be stored in memory.
+    store : MutableMapping or string
+        Store or path to directory in file system.
     overwrite : bool, optional
         If True, delete any pre-existing data in `store` at `path` before
         creating the group.
@@ -765,6 +774,8 @@ def group(store=None, overwrite=False, chunk_store=None, synchronizer=None):
         for storage of both chunks and metadata.
     synchronizer : object, optional
         Array synchronizer.
+    path : string, optional
+        Group path.
 
     Returns
     -------
@@ -791,33 +802,34 @@ def group(store=None, overwrite=False, chunk_store=None, synchronizer=None):
 
     """
 
-    # ensure store
-    if store is None:
-        store = DictStore()
+    # handle polymorphic store arg
+    store = _handle_store_arg(store)
 
     # require group
     if overwrite or not contains_group(store):
-        init_group(store, overwrite=overwrite, chunk_store=chunk_store)
+        init_group(store, overwrite=overwrite, chunk_store=chunk_store,
+                   path=path)
 
     return Group(store, read_only=False, chunk_store=chunk_store,
-                 synchronizer=synchronizer)
+                 synchronizer=synchronizer, path=path)
 
 
-def open_group(path, mode='a', synchronizer=None):
-    """Convenience function to instantiate a group stored in a directory on
-    the file system.
+def open_group(store=None, mode='a', synchronizer=None, path=None):
+    """Open a group using mode-like semantics.
 
     Parameters
     ----------
-    path : string
-        Path to directory in file system in which to store the group.
+    store : MutableMapping or string
+        Store or path to directory in file system.
     mode : {'r', 'r+', 'a', 'w', 'w-'}
         Persistence mode: 'r' means read only (must exist); 'r+' means
         read/write (must exist); 'a' means read/write (create if doesn't
         exist); 'w' means create (overwrite if exists); 'w-' means create
         (fail if exists).
     synchronizer : object, optional
         Array synchronizer.
+    path : string, optional
+        Group path.
 
     Returns
     -------
@@ -843,35 +855,36 @@ def open_group(path, mode='a', synchronizer=None):
 
     """
 
-    # setup store
-    store = DirectoryStore(path)
+    # handle polymorphic store arg
+    store = _handle_store_arg(store)
 
     # ensure store is initialized
 
     if mode in ['r', 'r+']:
-        if contains_array(store):
+        if contains_array(store, path=path):
             raise ValueError('store contains array')
-        elif not contains_group(store):
+        elif not contains_group(store, path=path):
             raise ValueError('group does not exist')
 
     elif mode == 'w':
-        init_group(store, overwrite=True)
+        init_group(store, overwrite=True, path=path)
 
     elif mode == 'a':
-        if contains_array(store):
+        if contains_array(store, path=path):
             raise ValueError('store contains array')
-        if not contains_group(store):
-            init_group(store)
+        if not contains_group(store, path=path):
+            init_group(store, path=path)
 
     elif mode in ['w-', 'x']:
-        if contains_array(store):
+        if contains_array(store, path=path):
             raise ValueError('store contains array')
-        elif contains_group(store):
+        elif contains_group(store, path=path):
             raise ValueError('store contains group')
         else:
-            init_group(store)
+            init_group(store, path=path)
 
     # determine read only status
     read_only = mode == 'r'
 
-    return Group(store, read_only=read_only, synchronizer=synchronizer)
+    return Group(store, read_only=read_only, synchronizer=synchronizer,
+                 path=path)

zarr/tests/test_creation.py

@@ -101,10 +101,10 @@ def test_full():
 
 def test_open_array():
 
-    path = 'example'
+    store = 'example'
 
     # mode == 'w'
-    z = open_array(path, mode='w', shape=100, chunks=10)
+    z = open_array(store, mode='w', shape=100, chunks=10)
     z[:] = 42
     assert_is_instance(z, Array)
     assert_is_instance(z.store, DirectoryStore)
@@ -119,15 +119,15 @@ def test_open_array():
             open_array('doesnotexist', mode=mode)
         with assert_raises(ValueError):
             open_array('example_group', mode=mode)
-    z = open_array(path, mode='r')
+    z = open_array(store, mode='r')
     assert_is_instance(z, Array)
     assert_is_instance(z.store, DirectoryStore)
     eq((100,), z.shape)
     eq((10,), z.chunks)
     assert_array_equal(np.full(100, fill_value=42), z[:])
     with assert_raises(PermissionError):
         z[:] = 43
-    z = open_array(path, mode='r+')
+    z = open_array(store, mode='r+')
     assert_is_instance(z, Array)
     assert_is_instance(z.store, DirectoryStore)
     eq((100,), z.shape)
@@ -137,8 +137,8 @@ def test_open_array():
     assert_array_equal(np.full(100, fill_value=43), z[:])
 
     # mode == 'a'
-    shutil.rmtree(path)
-    z = open_array(path, mode='a', shape=100, chunks=10)
+    shutil.rmtree(store)
+    z = open_array(store, mode='a', shape=100, chunks=10)
     z[:] = 42
     assert_is_instance(z, Array)
     assert_is_instance(z.store, DirectoryStore)
@@ -150,23 +150,28 @@ def test_open_array():
 
     # mode in 'w-', 'x'
     for mode in 'w-', 'x':
-        shutil.rmtree(path)
-        z = open_array(path, mode=mode, shape=100, chunks=10)
+        shutil.rmtree(store)
+        z = open_array(store, mode=mode, shape=100, chunks=10)
         z[:] = 42
         assert_is_instance(z, Array)
         assert_is_instance(z.store, DirectoryStore)
         eq((100,), z.shape)
         eq((10,), z.chunks)
         assert_array_equal(np.full(100, fill_value=42), z[:])
         with assert_raises(ValueError):
-            open_array(path, mode=mode)
+            open_array(store, mode=mode)
         with assert_raises(ValueError):
             open_array('example_group', mode=mode)
 
     # with synchronizer
-    z = open_array(path, synchronizer=ThreadSynchronizer())
+    z = open_array(store, synchronizer=ThreadSynchronizer())
     assert_is_instance(z, Array)
 
+    # with path
+    z = open_array(store, shape=100, path='foo/bar', mode='w')
+    assert_is_instance(z, Array)
+    eq('foo/bar', z.path)
+
 
 def test_empty_like():
     # zarr array