migrate compression metadata

Author: alimanfoo

zarr/core.py

@@ -729,11 +729,10 @@ def __repr__(self):
 
         # storage size info
         r += '\n  nbytes: %s' % human_readable_size(self.nbytes)
-        if self.compression:
-            if self.nbytes_stored > 0:
-                r += '; nbytes_stored: %s' % human_readable_size(
-                    self.nbytes_stored)
-                r += '; ratio: %.1f' % (self.nbytes / self.nbytes_stored)
+        if self.nbytes_stored > 0:
+            r += '; nbytes_stored: %s' % human_readable_size(
+                self.nbytes_stored)
+            r += '; ratio: %.1f' % (self.nbytes / self.nbytes_stored)
         n_chunks = reduce(operator.mul, self.cdata_shape)
         r += '; initialized: %s/%s' % (self.initialized, n_chunks)
 

zarr/storage.py

@@ -792,10 +792,27 @@ def migrate_1to2(store):
     from zarr import meta_v1
     meta = meta_v1.decode_metadata(store['meta'])
     del store['meta']
+
+    # add empty filters
     meta['filters'] = None
-    if meta['compression'] and meta['compression'].lower() == 'none':
-        meta['compression'] = None
-        meta['compression_opts'] = None
+
+    # migration compression metadata
+    compression = meta['compression']
+    if compression is None or compression == 'none':
+        compressor_config = None
+    else:
+        compression_opts = meta['compression_opts']
+        codec_cls = codec_registry[compression]
+        if isinstance(compression_opts, dict):
+            compressor = codec_cls(**compression_opts)
+        else:
+            compressor = codec_cls(compression_opts)
+        compressor_config = compressor.get_config()
+    meta['compressor'] = compressor_config
+    del meta['compression']
+    del meta['compression_opts']
+
+    # store migrated metadata
     store[array_meta_key] = encode_array_metadata(meta)
 
     # migrate user attributes

zarr/tests/test_storage.py

@@ -20,7 +20,7 @@
     ZARR_FORMAT, decode_group_metadata, encode_group_metadata
 from zarr.compat import text_type
 from zarr.storage import default_compressor
-from zarr.codecs import Zlib
+from zarr.codecs import Zlib, Blosc
 
 
 class StoreTests(object):
@@ -670,18 +670,38 @@ def test_migrate_1to2():
 
     # check results
     assert 'meta' not in store
-    assert '.zarray' in store
+    assert array_meta_key in store
     assert 'attrs' not in store
-    assert '.zattrs' in store
+    assert attrs_key in store
     meta_migrated = decode_array_metadata(store[array_meta_key])
     eq(2, meta_migrated['zarr_format'])
+
     # preserved fields
-    for f in 'shape', 'chunks', 'dtype', 'compression', 'compression_opts', \
-             'fill_value', 'order':
+    for f in 'shape', 'chunks', 'dtype', 'fill_value', 'order':
         eq(meta[f], meta_migrated[f])
+
     # migrate should have added empty filters field
     assert_is_none(meta_migrated['filters'])
 
+    # check compression and compression_opts migrated to compressor
+    assert 'compression' not in meta_migrated
+    assert 'compression_opts' not in meta_migrated
+    eq(meta_migrated['compressor'], Zlib(1).get_config())
+
+    # check dict compression_opts
+    store = dict()
+    meta['compression'] = 'blosc'
+    meta['compression_opts'] = dict(cname='lz4', clevel=5, shuffle=1)
+    meta_json = meta_v1.encode_metadata(meta)
+    store['meta'] = meta_json
+    store['attrs'] = json.dumps(dict()).encode('ascii')
+    migrate_1to2(store)
+    meta_migrated = decode_array_metadata(store[array_meta_key])
+    assert 'compression' not in meta_migrated
+    assert 'compression_opts' not in meta_migrated
+    eq(meta_migrated['compressor'],
+       Blosc(cname='lz4', clevel=5, shuffle=1).get_config())
+
     # check 'none' compression is migrated to None (null in JSON)
     store = dict()
     meta['compression'] = 'none'
@@ -690,5 +710,6 @@ def test_migrate_1to2():
     store['attrs'] = json.dumps(dict()).encode('ascii')
     migrate_1to2(store)
     meta_migrated = decode_array_metadata(store[array_meta_key])
-    assert_is_none(meta_migrated['compression'])
-    assert_is_none(meta_migrated['compression_opts'])
+    assert 'compression' not in meta_migrated
+    assert 'compression_opts' not in meta_migrated
+    assert_is_none(meta_migrated['compressor'])