Remove optimisation and finish benchmark

Author: lshaw8317

bench/ndarray/fancy_index.py

@@ -16,23 +16,28 @@
 import zarr
 import h5py
 import pickle
+import os
 plt.rcParams.update({'text.usetex':False,'font.serif': ['cm'],'font.size':16})
 plt.rcParams['figure.dpi'] = 1000
 plt.rcParams['savefig.dpi'] = 1000
 plt.rc('text', usetex=False)
 plt.rc('font',**{'serif':['cm']})
 plt.style.use('seaborn-v0_8-paper')
 
-NUMPY_BLOSC = True
-NUMPY_BLOSC_ZARR = False
+NUMPY_BLOSC = True # activate NUMPY and BLOSC tests
+NUMPY_BLOSC_ZARR = False # activate NUMPY, BLOSC and Zarr tests
+# default if both are false is to run tests for Numpy, Blosc, Zarr and HDF5
 
 def genarray(r, ndims=1, verbose=True):
     d = int((r*2**30/8)**(1/ndims))
     shape = (d,) * ndims
     chunks = (d // 4,) * ndims
     blocks = (max(d // 10, 1),) * ndims
     t = time.time()
-    arr = blosc2.ones(shape=shape, dtype=np.int64)
+    if os.path.exists(f'linspace{r}.b2nd'):
+        arr = blosc2.open(urlpath=f'linspace{r}.b2nd')
+    else:
+        arr = blosc2.linspace(0, 1000, num=np.prod(shape), shape=shape, dtype=np.float64, urlpath=f'linspace{r}.b2nd', mode='w')
     t = time.time() - t
     if verbose:
         print(f"Array shape: {arr.shape}")
@@ -41,15 +46,15 @@ def genarray(r, ndims=1, verbose=True):
     return arr
 
 
-sizes = np.int64(np.array([1, 2, 4, 8, 16]))
+sizes = np.int64(np.array([1, 2, 4, 8, 16, 24]))
 rng = np.random.default_rng()
 blosctimes = []
 nptimes = []
 zarrtimes = []
 h5pytimes = []
 x = np.arange(len(sizes))
 width = 0.2
-labs = 'NumpyBlosc2' if NUMPY_BLOSC else 'NumpyBlosc2ZarrHDF5' 
+labs = 'NumpyBlosc2' if NUMPY_BLOSC else 'NumpyBlosc2ZarrHDF5'
 labs = 'NumpyBlosc2Zarr' if NUMPY_BLOSC_ZARR else labs
 try:
     with open(f"results{labs}.pkl", 'rb') as f:
@@ -118,8 +123,8 @@ def timer(arr, row=row, col=col):
             z_test[:] = arr
             zarrtimes += [timer(z_test)]
             with h5py.File('my_hdf5_file.h5', 'w') as f:
-                dset = f.create_dataset("init", data=arr)
-                h5pytimes += [timer(dset)]
+                    dset = f.create_dataset("init", data=arr)
+                    h5pytimes += [timer(dset)]
 
     blosctimes = np.array(blosctimes)
     nptimes = np.array(nptimes)
@@ -135,8 +140,8 @@ def timer(arr, row=row, col=col):
             c = ['b', 'r', 'g', 'm'][i]
             mean = times.mean(axis=1)
             err = (mean - times.min(axis=1), times.max(axis=1)-mean)
-            plt.bar(x + w, mean , width, color=c, label=label, yerr=err, capsize=5, ecolor='k', 
-            error_kw=dict(lw=2, capthick=2, ecolor='k'))            
+            plt.bar(x + w, mean , width, color=c, label=label, yerr=err, capsize=5, ecolor='k',
+            error_kw=dict(lw=2, capthick=2, ecolor='k'))
             labs+=label
 
     with open(f"results{labs}.pkl", 'wb') as f:
@@ -147,7 +152,8 @@ def timer(arr, row=row, col=col):
 plt.xticks(x-width, np.round(sizes, 2))
 plt.ylabel("Time (s)")
 plt.title('Fancy indexing performance comparison')
-plt.ylim([0,10])
+# plt.ylim([0,10])
+plt.gca().set_yscale('log')
 plt.savefig(f'plots/fancyIdx{labs}.png', format="png")
 plt.show()
 

src/blosc2/ndarray.py

@@ -1442,9 +1442,6 @@ def T(self):
     def get_fselection_numpy(self, key):
         # TODO: Make this faster for broadcasted keys
         shape = self.shape
-        if math.prod(shape) * self.dtype.itemsize < blosc2.MAX_FAST_PATH_SIZE:
-            # just load whole array into memory and do numpy indexing
-            return self[:][key]
         chunks = self.chunks
         _slice = ndindex.ndindex(key).expand(shape)
         chunk_size = ndindex.ChunkSize(chunks)
@@ -1466,20 +1463,20 @@ def get_fselection_numpy(self, key):
         return out
 
     def get_oselection_numpy(self, key):
-        '''
+        """
         Select independently from self along axes specified in key. Key must be same length as self shape.
         See Zarr https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#orthogonal-indexing.
-        '''
+        """
         shape = tuple(len(k) for k in key) + self.shape[len(key) :]
         # Create the array to store the result
         arr = np.empty(shape, dtype=self.dtype)
         return super().get_oindex_numpy(arr, key)
 
     def set_oselection_numpy(self, key, arr: np.ndarray):
-        '''
+        """
         Select independently from self along axes specified in key and set to entries in arr. Key must be same length as self shape.
         See Zarr https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#orthogonal-indexing.
-        '''
+        """
         return super().set_oindex_numpy(key, arr)
 
     def __getitem__(  # noqa: C901
@@ -4409,7 +4406,7 @@ def __init__(self, array: NDArray):
 
     # TODO: all this
     def __getitem__(self, selection) -> np.ndarray:
-        return NotImplementedError 
+        return NotImplementedError
 
     def __setitem__(self, selection, input) -> np.ndarray:
         return NotImplementedError