fix bug in Array pickle/unpickle with cache_attrs

Author: alimanfoo

zarr/core.py

@@ -1924,7 +1924,7 @@ def hexdigest(self, hashname="sha1"):
 
     def __getstate__(self):
         return (self._store, self._path, self._read_only, self._chunk_store,
-                self._synchronizer, self._cache_metadata)
+                self._synchronizer, self._cache_metadata, self._attrs.cache)
 
     def __setstate__(self, state):
         self.__init__(*state)

zarr/tests/test_core.py

@@ -79,8 +79,11 @@ def test_array_init(self):
     def create_array(self, read_only=False, **kwargs):
         store = dict()
         kwargs.setdefault('compressor', Zlib(level=1))
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_nbytes_stored(self):
 
@@ -655,7 +658,8 @@ def test_read_only(self):
 
     def test_pickle(self):
 
-        z = self.create_array(shape=1000, chunks=100, dtype=int)
+        z = self.create_array(shape=1000, chunks=100, dtype=int, cache_metadata=False,
+                              cache_attrs=False)
         z[:] = np.random.randint(0, 1000, 1000)
         z2 = pickle.loads(pickle.dumps(z))
         eq(z.shape, z2.shape)
@@ -664,6 +668,8 @@ def test_pickle(self):
         if z.compressor:
             eq(z.compressor.get_config(), z2.compressor.get_config())
         eq(z.fill_value, z2.fill_value)
+        eq(z._cache_metadata, z2._cache_metadata)
+        eq(z.attrs.cache, z2.attrs.cache)
         assert_array_equal(z[:], z2[:])
 
     def test_np_ufuncs(self):
@@ -1081,8 +1087,11 @@ class TestArrayWithPath(TestArray):
     @staticmethod
     def create_array(read_only=False, **kwargs):
         store = dict()
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, path='foo/bar', **kwargs)
-        return Array(store, path='foo/bar', read_only=read_only)
+        return Array(store, path='foo/bar', read_only=read_only,
+                     cache_metadata=cache_metadata, cache_attrs=cache_attrs)
 
     def test_hexdigest(self):
         # Check basic 1-D array
@@ -1133,8 +1142,11 @@ def create_array(read_only=False, **kwargs):
         store = dict()
         # separate chunk store
         chunk_store = dict()
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, chunk_store=chunk_store, **kwargs)
-        return Array(store, read_only=read_only, chunk_store=chunk_store)
+        return Array(store, read_only=read_only, chunk_store=chunk_store,
+                     cache_metadata=cache_metadata, cache_attrs=cache_attrs)
 
     def test_hexdigest(self):
         # Check basic 1-D array
@@ -1184,9 +1196,12 @@ def create_array(read_only=False, **kwargs):
         path = mkdtemp()
         atexit.register(shutil.rmtree, path)
         store = DirectoryStore(path)
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         kwargs.setdefault('compressor', Zlib(1))
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_nbytes_stored(self):
 
@@ -1206,9 +1221,12 @@ def create_array(read_only=False, **kwargs):
         path = mkdtemp()
         atexit.register(shutil.rmtree, path)
         store = NestedDirectoryStore(path)
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         kwargs.setdefault('compressor', Zlib(1))
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
 
 class TestArrayWithDBMStore(TestArray):
@@ -1218,9 +1236,12 @@ def create_array(read_only=False, **kwargs):
         path = mktemp(suffix='.anydbm')
         atexit.register(atexit_rmglob, path + '*')
         store = DBMStore(path, flag='n')
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         kwargs.setdefault('compressor', Zlib(1))
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_attrs=cache_attrs,
+                     cache_metadata=cache_metadata)
 
     def test_nbytes_stored(self):
         pass  # not implemented
@@ -1236,9 +1257,12 @@ def create_array(read_only=False, **kwargs):
             path = mktemp(suffix='.dbm')
             atexit.register(os.remove, path)
             store = DBMStore(path, flag='n', open=bsddb3.btopen)
+            cache_metadata = kwargs.pop('cache_metadata', True)
+            cache_attrs = kwargs.pop('cache_attrs', True)
             kwargs.setdefault('compressor', Zlib(1))
             init_array(store, **kwargs)
-            return Array(store, read_only=read_only)
+            return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                         cache_attrs=cache_attrs)
 
         def test_nbytes_stored(self):
             pass  # not implemented
@@ -1257,9 +1281,12 @@ def create_array(read_only=False, **kwargs):
             store = LMDBStore(path, buffers=True)
         except ImportError:  # pragma: no cover
             raise SkipTest('lmdb not installed')
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         kwargs.setdefault('compressor', Zlib(1))
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_nbytes_stored(self):
         pass  # not implemented
@@ -1275,9 +1302,12 @@ def create_array(read_only=False, **kwargs):
             store = LMDBStore(path, buffers=False)
         except ImportError:  # pragma: no cover
             raise SkipTest('lmdb not installed')
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         kwargs.setdefault('compressor', Zlib(1))
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_nbytes_stored(self):
         pass  # not implemented
@@ -1288,8 +1318,11 @@ class TestArrayWithNoCompressor(TestArray):
     def create_array(self, read_only=False, **kwargs):
         store = dict()
         kwargs.setdefault('compressor', None)
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_hexdigest(self):
         # Check basic 1-D array
@@ -1321,8 +1354,11 @@ def create_array(self, read_only=False, **kwargs):
         store = dict()
         compressor = BZ2(level=1)
         kwargs.setdefault('compressor', compressor)
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_hexdigest(self):
         # Check basic 1-D array
@@ -1354,8 +1390,11 @@ def create_array(self, read_only=False, **kwargs):
         store = dict()
         compressor = Blosc(cname='zstd', clevel=1, shuffle=1)
         kwargs.setdefault('compressor', compressor)
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_hexdigest(self):
         # Check basic 1-D array
@@ -1392,8 +1431,11 @@ def create_array(self, read_only=False, **kwargs):
             store = dict()
             compressor = LZMA(preset=1)
             kwargs.setdefault('compressor', compressor)
+            cache_metadata = kwargs.pop('cache_metadata', True)
+            cache_attrs = kwargs.pop('cache_attrs', True)
             init_array(store, **kwargs)
-            return Array(store, read_only=read_only)
+            return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                         cache_attrs=cache_attrs)
 
         def test_hexdigest(self):
             # Check basic 1-D array
@@ -1432,8 +1474,11 @@ def create_array(read_only=False, **kwargs):
         kwargs.setdefault('filters', filters)
         compressor = Zlib(1)
         kwargs.setdefault('compressor', compressor)
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_attrs=cache_attrs,
+                     cache_metadata=cache_metadata)
 
     def test_hexdigest(self):
         # Check basic 1-D array
@@ -1555,8 +1600,11 @@ class TestArrayWithCustomMapping(TestArray):
     def create_array(read_only=False, **kwargs):
         store = CustomMapping()
         kwargs.setdefault('compressor', Zlib(1))
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_nbytes_stored(self):
         z = self.create_array(shape=1000, chunks=100)
@@ -1571,12 +1619,14 @@ class TestArrayNoCache(TestArray):
     def create_array(read_only=False, **kwargs):
         store = dict()
         kwargs.setdefault('compressor', Zlib(level=1))
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, **kwargs)
-        return Array(store, read_only=read_only, cache_metadata=False,
-                     cache_attrs=False)
+        return Array(store, read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def test_cache_metadata(self):
-        a1 = self.create_array(shape=100, chunks=10, dtype='i1')
+        a1 = self.create_array(shape=100, chunks=10, dtype='i1', cache_metadata=False)
         a2 = Array(a1.store, cache_metadata=True)
         eq(a1.shape, a2.shape)
         eq(a1.size, a2.size)
@@ -1616,7 +1666,7 @@ def test_cache_metadata(self):
         eq(30, a2.nchunks)
 
     def test_cache_attrs(self):
-        a1 = self.create_array(shape=100, chunks=10, dtype='i1')
+        a1 = self.create_array(shape=100, chunks=10, dtype='i1', cache_attrs=False)
         a2 = Array(a1.store, cache_attrs=True)
         eq(a1.attrs.asdict(), a2.attrs.asdict())
 

zarr/tests/test_sync.py

@@ -102,9 +102,12 @@ class TestArrayWithThreadSynchronizer(TestArray, MixinArraySyncTests):
 
     def create_array(self, read_only=False, **kwargs):
         store = dict()
+        cache_metadata = kwargs.pop('cache_metadata', True)
+        cache_attrs = kwargs.pop('cache_attrs', True)
         init_array(store, **kwargs)
         return Array(store, synchronizer=ThreadSynchronizer(),
-                     read_only=read_only)
+                     read_only=read_only, cache_metadata=cache_metadata,
+                     cache_attrs=cache_attrs)
 
     def create_pool(self):
         pool = ThreadPool(cpu_count())
@@ -140,12 +143,14 @@ def create_array(self, read_only=False, **kwargs):
         path = tempfile.mkdtemp()
         atexit.register(atexit_rmtree, path)
         store = DirectoryStore(path)
+        cache_metadata = kwargs.pop('cache_metadata', False)
+        cache_attrs = kwargs.pop('cache_attrs', False)
         init_array(store, **kwargs)
         sync_path = tempfile.mkdtemp()
         atexit.register(atexit_rmtree, sync_path)
         synchronizer = ProcessSynchronizer(sync_path)
-        return Array(store, synchronizer=synchronizer,
-                     read_only=read_only, cache_metadata=False)
+        return Array(store, synchronizer=synchronizer, read_only=read_only,
+                     cache_metadata=cache_metadata, cache_attrs=cache_attrs)
 
     def create_pool(self):
         pool = ProcessPool(processes=cpu_count())