implement overwrite properly; resolves #71

Author: alimanfoo

zarr/creation.py

@@ -8,7 +8,7 @@
 
 from zarr.core import Array
 from zarr.storage import DirectoryStore, init_array, contains_array, \
-    contains_group, default_compressor
+    contains_group, default_compressor, _require_parent_group
 from zarr.codecs import codec_registry
 
 

zarr/hierarchy.py

@@ -436,12 +436,10 @@ def create_group(self, name, overwrite=False):
     def _create_group_nosync(self, name, overwrite=False):
         path = self._item_path(name)
 
-        # create intermediate groups
-        self._require_parent_group(path, overwrite=overwrite)
-
         # create terminal group
         init_group(self._store, path=path, chunk_store=self._chunk_store,
                    overwrite=overwrite)
+
         return Group(self._store, path=path, read_only=self._read_only,
                      chunk_store=self._chunk_store,
                      synchronizer=self._synchronizer)
@@ -450,13 +448,15 @@ def create_groups(self, *names, **kwargs):
         """Convenience method to create multiple groups in a single call."""
         return tuple(self.create_group(name, **kwargs) for name in names)
 
-    def require_group(self, name):
+    def require_group(self, name, overwrite=False):
         """Obtain a sub-group, creating one if it doesn't exist.
 
         Parameters
         ----------
         name : string
             Group name.
+        overwrite : bool, optional
+            Overwrite any existing array with given `name` if present.
 
         Returns
         -------
@@ -473,20 +473,17 @@ def require_group(self, name):
 
         """
 
-        return self._write_op(self._require_group_nosync, name)
-
-    def _require_group_nosync(self, name):
+        return self._write_op(self._require_group_nosync, name,
+                              overwrite=overwrite)
 
+    def _require_group_nosync(self, name, overwrite=False):
         path = self._item_path(name)
 
-        # require all intermediate groups
-        segments = path.split('/')
-        for i in range(len(segments) + 1):
-            p = '/'.join(segments[:i])
-            if contains_array(self._store, p):
-                raise KeyError(name)
-            elif not contains_group(self._store, p):
-                init_group(self._store, path=p, chunk_store=self._chunk_store)
+        # create terminal group if necessary
+        if not contains_group(self._store, path):
+            init_group(store=self._store, path=path,
+                       chunk_store=self._chunk_store,
+                       overwrite=overwrite)
 
         return Group(self._store, path=path, read_only=self._read_only,
                      chunk_store=self._chunk_store,
@@ -496,16 +493,6 @@ def require_groups(self, *names):
         """Convenience method to require multiple groups in a single call."""
         return tuple(self.require_group(name) for name in names)
 
-    def _require_parent_group(self, path, overwrite=False):
-        segments = path.split('/')
-        for i in range(len(segments)):
-            p = '/'.join(segments[:i])
-            if contains_array(self._store, p):
-                init_group(self._store, path=p,
-                           chunk_store=self._chunk_store, overwrite=overwrite)
-            elif not contains_group(self._store, p):
-                init_group(self._store, path=p, chunk_store=self._chunk_store)
-
     def create_dataset(self, name, data=None, shape=None, chunks=None,
                        dtype=None, compressor='default', fill_value=None,
                        order='C', synchronizer=None, filters=None,
@@ -569,7 +556,6 @@ def _create_dataset_nosync(self, name, data=None, shape=None, chunks=None,
                                filters=None, overwrite=False, **kwargs):
 
         path = self._item_path(name)
-        self._require_parent_group(path, overwrite=overwrite)
 
         # determine synchronizer
         if synchronizer is None:
@@ -647,8 +633,6 @@ def create(self, name, **kwargs):
 
     def _create_nosync(self, name, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return create(store=self._store, path=path,
                       chunk_store=self._chunk_store, **kwargs)
@@ -660,8 +644,6 @@ def empty(self, name, **kwargs):
 
     def _empty_nosync(self, name, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return empty(store=self._store, path=path,
                      chunk_store=self._chunk_store, **kwargs)
@@ -673,8 +655,6 @@ def zeros(self, name, **kwargs):
 
     def _zeros_nosync(self, name, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return zeros(store=self._store, path=path,
                      chunk_store=self._chunk_store, **kwargs)
@@ -686,8 +666,6 @@ def ones(self, name, **kwargs):
 
     def _ones_nosync(self, name, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return ones(store=self._store, path=path,
                     chunk_store=self._chunk_store, **kwargs)
@@ -699,8 +677,6 @@ def full(self, name, fill_value, **kwargs):
 
     def _full_nosync(self, name, fill_value, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return full(store=self._store, path=path,
                     chunk_store=self._chunk_store,
@@ -713,8 +689,6 @@ def array(self, name, data, **kwargs):
 
     def _array_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return array(data, store=self._store, path=path,
                      chunk_store=self._chunk_store, **kwargs)
@@ -726,8 +700,6 @@ def empty_like(self, name, data, **kwargs):
 
     def _empty_like_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return empty_like(data, store=self._store, path=path,
                           chunk_store=self._chunk_store, **kwargs)
@@ -739,8 +711,6 @@ def zeros_like(self, name, data, **kwargs):
 
     def _zeros_like_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return zeros_like(data, store=self._store, path=path,
                           chunk_store=self._chunk_store, **kwargs)
@@ -752,8 +722,6 @@ def ones_like(self, name, data, **kwargs):
 
     def _ones_like_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return ones_like(data, store=self._store, path=path,
                          chunk_store=self._chunk_store, **kwargs)
@@ -765,8 +733,6 @@ def full_like(self, name, data, **kwargs):
 
     def _full_like_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        overwrite = kwargs.get('overwrite', False)
-        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return full_like(data, store=self._store, path=path,
                          chunk_store=self._chunk_store, **kwargs)
@@ -819,12 +785,8 @@ def group(store=None, overwrite=False, chunk_store=None, synchronizer=None):
         store = DictStore()
 
     # require group
-    if overwrite:
-        init_group(store, overwrite=True, chunk_store=chunk_store)
-    elif contains_array(store):
-        raise ValueError('store contains an array')
-    elif not contains_group(store):
-        init_group(store, chunk_store=chunk_store)
+    if overwrite or not contains_group(store):
+        init_group(store, overwrite=overwrite, chunk_store=chunk_store)
 
     return Group(store, read_only=False, chunk_store=chunk_store,
                  synchronizer=synchronizer)
@@ -887,7 +849,7 @@ def open_group(path, mode='a', synchronizer=None):
     elif mode == 'a':
         if contains_array(store):
             raise ValueError('store contains array')
-        elif not contains_group(store):
+        if not contains_group(store):
             init_group(store)
 
     elif mode in ['w-', 'x']:

zarr/storage.py

@@ -122,6 +122,19 @@ def getsize(store, path=None):
         return -1
 
 
+def _require_parent_group(path, store, chunk_store, overwrite):
+    path = normalize_storage_path(path)
+    if path:
+        segments = path.split('/')
+        for i in range(len(segments)):
+            p = '/'.join(segments[:i])
+            if contains_array(store, p):
+                _init_group_metadata(store, path=p, chunk_store=chunk_store,
+                                     overwrite=overwrite)
+            elif not contains_group(store, p):
+                _init_group_metadata(store, path=p, chunk_store=chunk_store)
+
+
 def init_array(store, shape, chunks, dtype=None, compressor='default',
                fill_value=None, order='C', overwrite=False, path=None,
                chunk_store=None, filters=None):
@@ -195,11 +208,12 @@ def init_array(store, shape, chunks, dtype=None, compressor='default',
 
     Initialize an array using a storage path::
 
+        >>> store = dict()
         >>> init_array(store, shape=100000000, chunks=1000000, dtype='i1',
-        ...            path='foo/bar')
+        ...            path='foo')
         >>> sorted(store.keys())
-        ['.zarray', '.zattrs', 'foo/bar/.zarray', 'foo/bar/.zattrs']
-        >>> print(str(store['foo/bar/.zarray'], 'ascii'))
+        ['.zattrs', '.zgroup', 'foo/.zarray', 'foo/.zattrs']
+        >>> print(str(store['foo/.zarray'], 'ascii'))
         {
             "chunks": [
                 1000000
@@ -231,6 +245,10 @@ def init_array(store, shape, chunks, dtype=None, compressor='default',
     # normalize path
     path = normalize_storage_path(path)
 
+    # ensure parent group initialized
+    _require_parent_group(path, store=store, chunk_store=chunk_store,
+                          overwrite=overwrite)
+
     # guard conditions
     if overwrite:
         # attempt to delete any pre-existing items in store
@@ -304,7 +322,18 @@ def init_group(store, overwrite=False, path=None, chunk_store=None):
 
     # normalize path
     path = normalize_storage_path(path)
-    
+
+    # ensure parent group initialized
+    _require_parent_group(path, store=store, chunk_store=chunk_store,
+                          overwrite=overwrite)
+
+    # initialise metadata
+    _init_group_metadata(store=store, overwrite=overwrite, path=path,
+                         chunk_store=chunk_store)
+
+
+def _init_group_metadata(store, overwrite=False, path=None, chunk_store=None):
+
     # guard conditions
     if overwrite:
         # attempt to delete any pre-existing items in store