add overwrite kwarg; resolves #71

Author: alimanfoo

docs/release.rst

@@ -1,6 +1,9 @@
 Release notes
 =============
 
+* Added ``overwrite`` keyword argument to array and group creation methods
+  on the :class:`zarr.hierarchy.Group` class.
+
 .. _release_2.0.1:
 
 2.0.1

zarr/creation.py

@@ -38,8 +38,8 @@ def create(shape, chunks=None, dtype=None, compressor='default',
     synchronizer : object, optional
         Array synchronizer.
     overwrite : bool, optional
-        If True, delete all pre-existing data in `store` before creating the
-        array.
+        If True, delete all pre-existing data in `store` at `path` before
+        creating the array.
     path : string, optional
         Path under which array is stored.
     chunk_store : MutableMapping, optional

zarr/hierarchy.py

@@ -9,7 +9,7 @@
 from zarr.attrs import Attributes
 from zarr.core import Array
 from zarr.storage import contains_array, contains_group, init_group, \
-    DictStore, DirectoryStore, group_meta_key, attrs_key, listdir
+    DictStore, DirectoryStore, group_meta_key, attrs_key, listdir, rmdir
 from zarr.creation import array, create, empty, zeros, ones, full, \
     empty_like, zeros_like, ones_like, full_like
 from zarr.util import normalize_storage_path, normalize_shape
@@ -406,13 +406,15 @@ def _write_op(self, f, *args, **kwargs):
             with self._synchronizer[group_meta_key]:
                 return f(*args, **kwargs)
 
-    def create_group(self, name):
+    def create_group(self, name, overwrite=False):
         """Create a sub-group.
 
         Parameters
         ----------
         name : string
             Group name.
+        overwrite : bool, optional
+            If True, overwrite any existing array with the given name.
 
         Returns
         -------
@@ -428,35 +430,25 @@ def create_group(self, name):
 
         """
 
-        return self._write_op(self._create_group_nosync, name)
-
-    def _create_group_nosync(self, name):
+        return self._write_op(self._create_group_nosync, name,
+                              overwrite=overwrite)
 
+    def _create_group_nosync(self, name, overwrite=False):
         path = self._item_path(name)
 
-        # require intermediate groups
-        segments = path.split('/')
-        for i in range(len(segments)):
-            p = '/'.join(segments[:i])
-            if contains_array(self._store, p):
-                raise KeyError(name)
-            elif not contains_group(self._store, p):
-                init_group(self._store, path=p, chunk_store=self._chunk_store)
+        # create intermediate groups
+        self._require_parent_group(path, overwrite=overwrite)
 
         # create terminal group
-        if contains_array(self._store, path):
-            raise KeyError(name)
-        if contains_group(self._store, path):
-            raise KeyError(name)
-        else:
-            init_group(self._store, path=path, chunk_store=self._chunk_store)
-            return Group(self._store, path=path, read_only=self._read_only,
-                         chunk_store=self._chunk_store,
-                         synchronizer=self._synchronizer)
+        init_group(self._store, path=path, chunk_store=self._chunk_store,
+                   overwrite=overwrite)
+        return Group(self._store, path=path, read_only=self._read_only,
+                     chunk_store=self._chunk_store,
+                     synchronizer=self._synchronizer)
 
-    def create_groups(self, *names):
+    def create_groups(self, *names, **kwargs):
         """Convenience method to create multiple groups in a single call."""
-        return tuple(self.create_group(name) for name in names)
+        return tuple(self.create_group(name, **kwargs) for name in names)
 
     def require_group(self, name):
         """Obtain a sub-group, creating one if it doesn't exist.
@@ -504,18 +496,20 @@ def require_groups(self, *names):
         """Convenience method to require multiple groups in a single call."""
         return tuple(self.require_group(name) for name in names)
 
-    def _require_parent_group(self, path):
+    def _require_parent_group(self, path, overwrite=False):
         segments = path.split('/')
         for i in range(len(segments)):
             p = '/'.join(segments[:i])
             if contains_array(self._store, p):
-                raise KeyError(path)
+                init_group(self._store, path=p,
+                           chunk_store=self._chunk_store, overwrite=overwrite)
             elif not contains_group(self._store, p):
                 init_group(self._store, path=p, chunk_store=self._chunk_store)
 
     def create_dataset(self, name, data=None, shape=None, chunks=None,
                        dtype=None, compressor='default', fill_value=None,
-                       order='C', synchronizer=None, filters=None, **kwargs):
+                       order='C', synchronizer=None, filters=None,
+                       overwrite=False, **kwargs):
         """Create an array.
 
         Parameters
@@ -540,7 +534,10 @@ def create_dataset(self, name, data=None, shape=None, chunks=None,
         synchronizer : zarr.sync.ArraySynchronizer, optional
             Array synchronizer.
         filters : sequence of Codecs, optional
-            Sequence of filters to use to encode chunk data prior to compression.
+            Sequence of filters to use to encode chunk data prior to
+            compression.
+        overwrite : bool, optional
+            If True, replace any existing array or group with the given name.
 
         Returns
         -------
@@ -564,21 +561,15 @@ def create_dataset(self, name, data=None, shape=None, chunks=None,
                               shape=shape, chunks=chunks, dtype=dtype,
                               compressor=compressor, fill_value=fill_value,
                               order=order, synchronizer=synchronizer,
-                              filters=filters, **kwargs)
+                              filters=filters, overwrite=overwrite, **kwargs)
 
     def _create_dataset_nosync(self, name, data=None, shape=None, chunks=None,
                                dtype=None, compressor='default',
                                fill_value=None, order='C', synchronizer=None,
-                               filters=None, **kwargs):
+                               filters=None, overwrite=False, **kwargs):
 
         path = self._item_path(name)
-        self._require_parent_group(path)
-
-        # guard conditions
-        if contains_array(self._store, path):
-            raise KeyError(name)
-        if contains_group(self._store, path):
-            raise KeyError(name)
+        self._require_parent_group(path, overwrite=overwrite)
 
         # determine synchronizer
         if synchronizer is None:
@@ -590,15 +581,16 @@ def _create_dataset_nosync(self, name, data=None, shape=None, chunks=None,
                       compressor=compressor, fill_value=fill_value,
                       order=order, synchronizer=synchronizer,
                       store=self._store, path=path,
-                      chunk_store=self._chunk_store, filters=filters, **kwargs)
+                      chunk_store=self._chunk_store, filters=filters,
+                      overwrite=overwrite, **kwargs)
 
         else:
             a = create(shape=shape, chunks=chunks, dtype=dtype,
                        compressor=compressor, fill_value=fill_value,
                        order=order, synchronizer=synchronizer,
                        store=self._store, path=path,
                        chunk_store=self._chunk_store, filters=filters,
-                       **kwargs)
+                       overwrite=overwrite, **kwargs)
 
         return a
 
@@ -655,7 +647,8 @@ def create(self, name, **kwargs):
 
     def _create_nosync(self, name, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return create(store=self._store, path=path,
                       chunk_store=self._chunk_store, **kwargs)
@@ -667,7 +660,8 @@ def empty(self, name, **kwargs):
 
     def _empty_nosync(self, name, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return empty(store=self._store, path=path,
                      chunk_store=self._chunk_store, **kwargs)
@@ -679,7 +673,8 @@ def zeros(self, name, **kwargs):
 
     def _zeros_nosync(self, name, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return zeros(store=self._store, path=path,
                      chunk_store=self._chunk_store, **kwargs)
@@ -691,7 +686,8 @@ def ones(self, name, **kwargs):
 
     def _ones_nosync(self, name, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return ones(store=self._store, path=path,
                     chunk_store=self._chunk_store, **kwargs)
@@ -703,7 +699,8 @@ def full(self, name, fill_value, **kwargs):
 
     def _full_nosync(self, name, fill_value, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return full(store=self._store, path=path,
                     chunk_store=self._chunk_store,
@@ -716,7 +713,8 @@ def array(self, name, data, **kwargs):
 
     def _array_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return array(data, store=self._store, path=path,
                      chunk_store=self._chunk_store, **kwargs)
@@ -728,7 +726,8 @@ def empty_like(self, name, data, **kwargs):
 
     def _empty_like_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return empty_like(data, store=self._store, path=path,
                           chunk_store=self._chunk_store, **kwargs)
@@ -740,7 +739,8 @@ def zeros_like(self, name, data, **kwargs):
 
     def _zeros_like_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return zeros_like(data, store=self._store, path=path,
                           chunk_store=self._chunk_store, **kwargs)
@@ -752,7 +752,8 @@ def ones_like(self, name, data, **kwargs):
 
     def _ones_like_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return ones_like(data, store=self._store, path=path,
                          chunk_store=self._chunk_store, **kwargs)
@@ -764,7 +765,8 @@ def full_like(self, name, data, **kwargs):
 
     def _full_like_nosync(self, name, data, **kwargs):
         path = self._item_path(name)
-        self._require_parent_group(path)
+        overwrite = kwargs.get('overwrite', False)
+        self._require_parent_group(path, overwrite=overwrite)
         kwargs.setdefault('synchronizer', self._synchronizer)
         return full_like(data, store=self._store, path=path,
                          chunk_store=self._chunk_store, **kwargs)

zarr/storage.py

@@ -238,9 +238,9 @@ def init_array(store, shape, chunks, dtype=None, compressor='default',
         if chunk_store is not None and chunk_store != store:
             rmdir(chunk_store, path)
     elif contains_array(store, path):
-        raise ValueError('store contains an array')
+        raise KeyError('path %r contains an array' % path)
     elif contains_group(store, path):
-        raise ValueError('store contains a group')
+        raise KeyError('path %r contains a group' % path)
 
     # normalize metadata
     shape = normalize_shape(shape)
@@ -312,9 +312,9 @@ def init_group(store, overwrite=False, path=None, chunk_store=None):
         if chunk_store is not None and chunk_store != store:
             rmdir(chunk_store, path)
     elif contains_array(store, path):
-        raise ValueError('store contains an array')
+        raise KeyError('path %r contains an array' % path)
     elif contains_group(store, path):
-        raise ValueError('store contains a group')
+        raise KeyError('path %r contains a group' % path)
 
     # initialize metadata
     # N.B., currently no metadata properties are needed, however there may
@@ -492,7 +492,7 @@ def getsize(self, path=None):
                 parent, key = self._get_parent(path)
                 value = parent[key]
             except KeyError:
-                raise ValueError('path not found: %r' % path)
+                raise KeyError('path %r not found' % path)
         else:
             value = self.root
 
@@ -673,7 +673,7 @@ def getsize(self, path=None):
                     size += os.path.getsize(child_fs_path)
             return size
         else:
-            raise ValueError('path not found: %r' % path)
+            raise KeyError('path %r not found' % path)
 
 
 # noinspection PyPep8Naming
@@ -806,7 +806,7 @@ def getsize(self, path=None):
                     info = zf.getinfo(path)
                     return info.compress_size
                 except KeyError:
-                    raise ValueError('path not found: %r' % path)
+                    raise KeyError('path %r not found' % path)
             else:
                 return 0
 

zarr/tests/test_hierarchy.py

@@ -342,6 +342,31 @@ def test_create_errors(self):
         with assert_raises(PermissionError):
             g.require_dataset('zzz', shape=100, chunks=10)
 
+    def test_create_overwrite(self):
+        try:
+            for method_name in 'create_dataset', 'create', 'empty', 'zeros', \
+                               'ones':
+                g = self.create_group()
+                getattr(g, method_name)('foo', shape=100, chunks=10)
+
+                # overwrite array with array
+                d = getattr(g, method_name)('foo', shape=200, chunks=20,
+                                            overwrite=True)
+                eq((200,), d.shape)
+                # overwrite array with group
+                g2 = g.create_group('foo', overwrite=True)
+                eq(0, len(g2))
+                # overwrite group with array
+                d = getattr(g, method_name)('foo', shape=300, chunks=30,
+                                            overwrite=True)
+                eq((300,), d.shape)
+                # overwrite array with group
+                d = getattr(g, method_name)('foo/bar', shape=400, chunks=40,
+                                            overwrite=True)
+                assert_is_instance(g['foo'], Group)
+        except NotImplementedError:
+            pass
+
     def test_getitem_contains_iterators(self):
         # setup
         g1 = self.create_group()