From 6b24a649acaabf4354709fce4e2a116b61a2c5bd Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 23 Oct 2026 14:26:08 +0100 Subject: [PATCH] Build arrays through non-generic constructors The generic constructors of Constant, Primitive, Struct, Chunked and Decimal arrays called the generic Array::from_parts_unchecked and Array::try_from_parts directly. Each crate that called them compiled its own copy of the from_parts functions and of the full DynArrayData vtable for the encoding. The generic constructors now call a private non-generic function, so vortex-array compiles each vtable once. ScalarFn had the same problem because new_from_validated_slots was #[inline]; it is not inlined now. vortex-ffi release cdylib __text: 37,223,740 -> 36,613,308 bytes (-610 KB). Copies of the Constant vtable: 22 -> 1, Primitive: 17 -> 1. Signed-off-by: Robert Kruszewski Co-Authored-By: Claude Opus 5.5 --- vortex-array/src/arrays/chunked/array.rs | 22 +++++++++++- vortex-array/src/arrays/constant/array.rs | 9 ++++- vortex-array/src/arrays/decimal/array.rs | 26 ++++++++++++-- .../src/arrays/primitive/array/mod.rs | 34 ++++++++++++++++--- vortex-array/src/arrays/scalar_fn/array.rs | 4 ++- vortex-array/src/arrays/struct_/array.rs | 28 ++++++++++++--- 6 files changed, 110 insertions(+), 13 deletions(-) diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 6c0169d7a7c..0fe9bd03949 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -215,7 +215,7 @@ impl Array { /// Constructs a new `ChunkedArray`. pub fn try_new(chunks: impl IntoIterator, dtype: DType) -> VortexResult { - Array::try_from_parts(Self::parts_from_chunks::(chunks, dtype)?) + Self::try_from_chunked_parts(Self::parts_from_chunks::(chunks, dtype)?) } pub fn rechunk( @@ -278,6 +278,26 @@ impl Array { pub unsafe fn new_unchecked(chunks: impl IntoIterator, dtype: DType) -> Self { let parts = Self::parts_from_chunks::(chunks, dtype) .vortex_expect("unchecked chunked construction cannot fail"); + unsafe { Self::from_chunked_parts_unchecked(parts) } + } + + /// Creates the array from parts after validation. + /// + /// The generic constructors call this function. It is not generic, so crates that call those + /// constructors do not each compile the `Chunked` array vtable. + fn try_from_chunked_parts(parts: ArrayParts) -> VortexResult { + Array::try_from_parts(parts) + } + + /// Creates the array from parts without validation. + /// + /// The generic constructors call this function. It is not generic, so crates that call those + /// constructors do not each compile the `Chunked` array vtable. + /// + /// # Safety + /// + /// The parts must be logically consistent. + unsafe fn from_chunked_parts_unchecked(parts: ArrayParts) -> Self { unsafe { Array::from_parts_unchecked(parts) } } } diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index 5b69648faf9..f4c6843ade3 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -45,7 +45,14 @@ impl Array { where S: Into, { - let scalar = scalar.into(); + Self::from_scalar(scalar.into(), len) + } + + /// Creates the array from a [`Scalar`]. + /// + /// The generic constructors call this function. It is not generic, so crates that call those + /// constructors do not each compile the `Constant` array vtable. + fn from_scalar(scalar: Scalar, len: usize) -> Self { let dtype = scalar.dtype().clone(); let data = ConstantData::new(scalar); unsafe { diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index eff2e5c8688..fdbced955c5 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -426,7 +426,9 @@ impl Array { let len = buffer.len(); let slots = DecimalData::make_slots(&validity, len); let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data, slots)) } + unsafe { + Self::from_decimal_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data, slots)) + } } /// Creates a new [`DecimalArray`] from a host-native buffer with validation. @@ -439,7 +441,27 @@ impl Array { let len = buffer.len(); let slots = DecimalData::make_slots(&validity, len); let data = DecimalData::try_new(buffer, decimal_dtype)?; - Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data, slots)) + Self::try_from_decimal_parts(ArrayParts::new(Decimal, dtype, len, data, slots)) + } + + /// Creates the array from parts without validation. + /// + /// The generic constructors call this function. It is not generic, so crates that call those + /// constructors do not each compile the `Decimal` array vtable. + /// + /// # Safety + /// + /// The parts must be logically consistent. + unsafe fn from_decimal_parts_unchecked(parts: ArrayParts) -> Self { + unsafe { Array::from_parts_unchecked(parts) } + } + + /// Creates the array from parts after validation. + /// + /// The generic constructors call this function. It is not generic, so crates that call those + /// constructors do not each compile the `Decimal` array vtable. + fn try_from_decimal_parts(parts: ArrayParts) -> VortexResult { + Array::try_from_parts(parts) } /// Creates a new [`DecimalArray`] from an iterator of values. diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index 417f4e184a6..14e46af8e9b 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -347,7 +347,11 @@ impl Array { let len = 0; let data = PrimitiveData::empty::(nullability); let slots = PrimitiveData::make_slots(&Validity::from(nullability), len); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data, slots)) } + unsafe { + Self::from_primitive_parts_unchecked(ArrayParts::new( + Primitive, dtype, len, data, slots, + )) + } } /// Creates a new `PrimitiveArray`. @@ -361,7 +365,11 @@ impl Array { let len = buffer.len(); let slots = PrimitiveData::make_slots(&validity, len); let data = PrimitiveData::new(buffer, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data, slots)) } + unsafe { + Self::from_primitive_parts_unchecked(ArrayParts::new( + Primitive, dtype, len, data, slots, + )) + } } /// Constructs a new `PrimitiveArray`. @@ -371,7 +379,9 @@ impl Array { let slots = PrimitiveData::make_slots(&validity, len); let data = PrimitiveData::try_new(buffer, validity)?; Ok(unsafe { - Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data, slots)) + Self::from_primitive_parts_unchecked(ArrayParts::new( + Primitive, dtype, len, data, slots, + )) }) } @@ -385,7 +395,23 @@ impl Array { let len = buffer.len(); let slots = PrimitiveData::make_slots(&validity, len); let data = unsafe { PrimitiveData::new_unchecked(buffer, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data, slots)) } + unsafe { + Self::from_primitive_parts_unchecked(ArrayParts::new( + Primitive, dtype, len, data, slots, + )) + } + } + + /// Creates the array from parts without validation. + /// + /// The generic constructors call this function. It is not generic, so crates that call those + /// constructors do not each compile the `Primitive` array vtable. + /// + /// # Safety + /// + /// The parts must be logically consistent. + unsafe fn from_primitive_parts_unchecked(parts: ArrayParts) -> Self { + unsafe { Array::from_parts_unchecked(parts) } } /// Create a new array from a buffer handle. diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 391ea9a03cd..44a1a816a9f 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -119,7 +119,9 @@ impl Array { /// # Safety /// The caller must ensure arity and child lengths were validated and /// "dtype" is scalar function's return dtype. - #[inline] + /// + /// This function is not inlined, so crates that construct scalar function arrays do not each + /// compile the `ScalarFn` array vtable. unsafe fn new_from_validated_slots( scalar_fn: ScalarFnRef, dtype: DType, diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 6d3052f1b25..002f3a687a6 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -294,7 +294,7 @@ impl Array { slots.push(Some(field)); } let dtype = StructFields::new(names, field_dtypes); - Array::try_from_parts(ArrayParts::new( + Self::try_from_struct_parts(ArrayParts::new( Struct, DType::Struct(dtype, validity.nullability()), length, @@ -317,7 +317,7 @@ impl Array { let outer_dtype = DType::Struct(dtype, validity.nullability()); let slots = make_struct_slots(fields, &validity, length); unsafe { - Array::from_parts_unchecked(ArrayParts::new( + Self::from_struct_parts_unchecked(ArrayParts::new( Struct, outer_dtype, length, @@ -336,7 +336,7 @@ impl Array { ) -> VortexResult { let outer_dtype = DType::Struct(dtype, validity.nullability()); let slots = make_struct_slots(fields, &validity, length); - Array::try_from_parts(ArrayParts::new( + Self::try_from_struct_parts(ArrayParts::new( Struct, outer_dtype, length, @@ -345,6 +345,26 @@ impl Array { )) } + /// Creates the array from parts after validation. + /// + /// The generic constructors call this function. It is not generic, so crates that call those + /// constructors do not each compile the `Struct` array vtable. + fn try_from_struct_parts(parts: ArrayParts) -> VortexResult { + Array::try_from_parts(parts) + } + + /// Creates the array from parts without validation. + /// + /// The generic constructors call this function. It is not generic, so crates that call those + /// constructors do not each compile the `Struct` array vtable. + /// + /// # Safety + /// + /// The parts must be logically consistent. + unsafe fn from_struct_parts_unchecked(parts: ArrayParts) -> Self { + unsafe { Array::from_parts_unchecked(parts) } + } + /// Construct a `StructArray` from named fields. pub fn from_fields>(items: &[(N, ArrayRef)]) -> VortexResult { Self::try_from_iter(items.iter().map(|(a, b)| (a, b.clone()))) @@ -462,7 +482,7 @@ impl Array { let new_dtype = struct_dtype.without_field(position).ok()?; let new_array = unsafe { - Array::from_parts_unchecked(ArrayParts::new( + Self::from_struct_parts_unchecked(ArrayParts::new( Struct, DType::Struct(new_dtype, self.dtype().nullability()), len,