Raise an exception when dtype exceeds size limit

Author: FrancescAlted

src/blosc2/blosc2_ext.pyx

@@ -74,6 +74,7 @@ cdef extern from "blosc2.h":
         BLOSC2_MAX_OVERHEAD
         BLOSC2_MAX_BUFFERSIZE
         BLOSC2_MAXBLOCKSIZE
+        BLOSC2_MAXTYPESIZE
         BLOSC_MAX_TYPESIZE
         BLOSC_MIN_BUFFERSIZE
 
@@ -536,7 +537,7 @@ ctypedef struct udf_udata:
     int64_t chunks_in_array[B2ND_MAX_DIM]
     int64_t blocks_in_chunk[B2ND_MAX_DIM]
 
-MAX_TYPESIZE = BLOSC_MAX_TYPESIZE
+MAX_TYPESIZE = BLOSC2_MAXTYPESIZE
 MAX_BUFFERSIZE = BLOSC2_MAX_BUFFERSIZE
 MAX_BLOCKSIZE = BLOSC2_MAXBLOCKSIZE
 MAX_OVERHEAD = BLOSC2_MAX_OVERHEAD

src/blosc2/ndarray.py

@@ -2776,6 +2776,11 @@ def _check_shape(shape):
         raise TypeError("shape should be a tuple or a list!")
     return shape
 
+def _check_dtype(dtype):
+    dtype = np.dtype(dtype)
+    if dtype.itemsize > blosc2.MAX_TYPESIZE:
+        raise ValueError(f"dtype itemsize {dtype.itemsize} is too large (>{blosc2.MAX_TYPESIZE})!")
+    return dtype
 
 def empty(shape: int | tuple | list, dtype: np.dtype | str | None = np.float64, **kwargs: Any) -> NDArray:
     """Create an empty array.
@@ -2822,7 +2827,7 @@ def empty(shape: int | tuple | list, dtype: np.dtype | str | None = np.float64,
     >>> array.dtype
     dtype('int32')
     """
-    dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
     shape = _check_shape(shape)
     kwargs = _check_ndarray_kwargs(**kwargs)
     chunks = kwargs.pop("chunks", None)
@@ -2856,7 +2861,7 @@ def uninit(shape: int | tuple | list, dtype: np.dtype | str = np.float64, **kwar
     >>> array.dtype
     dtype('float64')
     """
-    dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
     shape = _check_shape(shape)
     kwargs = _check_ndarray_kwargs(**kwargs)
     chunks = kwargs.pop("chunks", None)
@@ -2890,7 +2895,7 @@ def nans(shape: int | tuple | list, dtype: np.dtype | str = np.float64, **kwargs
     >>> array.dtype
     dtype('float64')
     """
-    dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
     shape = _check_shape(shape)
     kwargs = _check_ndarray_kwargs(**kwargs)
     chunks = kwargs.pop("chunks", None)
@@ -2930,7 +2935,7 @@ def zeros(shape: int | tuple | list, dtype: np.dtype | str = np.float64, **kwarg
     >>> array.dtype
     dtype('float64')
     """
-    dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
     shape = _check_shape(shape)
     kwargs = _check_ndarray_kwargs(**kwargs)
     chunks = kwargs.pop("chunks", None)
@@ -2990,6 +2995,7 @@ def full(
         dtype = np.dtype(type(fill_value))
     else:
         dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
     shape = _check_shape(shape)
     kwargs = _check_ndarray_kwargs(**kwargs)
     chunks = kwargs.pop("chunks", None)
@@ -3101,8 +3107,7 @@ def arange_fill(inputs, output, offset):
         # Check that the shape is consistent with the start, stop and step values
         if math.prod(shape) != int((stop - start) / step):
             raise ValueError("The shape is not consistent with the start, stop and step values")
-    # Check dtype
-    dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
 
     if is_inside_new_expr():
         # We already have the dtype and shape, so return immediately
@@ -3170,8 +3175,7 @@ def linspace_fill(inputs, output, offset):
 
     if not shape:
         shape = (num,)
-    # Check dtype
-    dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
 
     if is_inside_new_expr():
         # We already have the dtype and shape, so return immediately
@@ -3230,8 +3234,7 @@ def fill_eye(inputs, output: np.array, offset: tuple):
     if M is None:
         M = N
     shape = (N, M)
-    # Check dtype
-    dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
 
     if is_inside_new_expr():
         # We already have the dtype and shape, so return immediately
@@ -3284,8 +3287,7 @@ def iter_fill(inputs, output, offset):
         (iterable,) = inputs
         output[:] = np.fromiter(iterable, dtype=output.dtype, count=nout).reshape(output.shape)
 
-    # Check dtype
-    dtype = np.dtype(dtype)
+    dtype = _check_dtype(dtype)
 
     if is_inside_new_expr():
         # We already have the dtype and shape, so return immediately