Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 81 additions & 5 deletions encodings/fastlanes/src/bitpacking/compute/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,20 @@ use vortex_array::ArrayRef;
use vortex_array::ArrayView;
use vortex_array::ExecutionCtx;
use vortex_array::IntoArray;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::arrays::primitive::PrimitiveArrayExt;
use vortex_array::builders::PrimitiveBuilder;
use vortex_array::builtins::ArrayBuiltins;
use vortex_array::dtype::DType;
use vortex_array::dtype::NativePType;
use vortex_array::dtype::PType;
use vortex_array::match_each_integer_ptype;
use vortex_array::match_each_unsigned_integer_ptype;
use vortex_array::scalar_fn::fns::cast::CastKernel;
use vortex_array::scalar_fn::fns::cast::CastReduce;
use vortex_array::validity::Validity;
use vortex_error::VortexResult;

use crate::BitPackedArray;
use crate::bitpacking::BitPacked;
use crate::bitpacking::BitWidthsView;
use crate::bitpacking::array::BitPackedArrayExt;
Expand Down Expand Up @@ -53,6 +57,39 @@ fn build_with_validity(
.into_array())
}

/// The array with the unsigned ptype of the same width.
///
/// The packed values and patches are never negative, so they keep their value. Only the patch
/// values are reinterpreted. The packed buffer is shared.
fn unsigned_view_of(
array: ArrayView<'_, BitPacked>,
bit_width: u8,
ctx: &mut ExecutionCtx,
) -> VortexResult<BitPackedArray> {
let ptype = array.dtype().as_ptype().to_unsigned();
let patches = array
.patches()
.map(|patches| {
patches.map_values(|values| {
Ok(values
.execute::<PrimitiveArray>(ctx)?
.reinterpret_cast(ptype)
.into_array())
})
})
.transpose()?;

BitPacked::try_new(
array.packed().clone(),
ptype,
array.validity()?,
patches,
bit_width,
array.len(),
array.offset(),
)
}

impl CastReduce for BitPacked {
fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult<Option<ArrayRef>> {
let BitWidthsView::Global(bit_width) = array.bit_widths() else {
Expand Down Expand Up @@ -107,16 +144,36 @@ impl CastKernel for BitPacked {
.validity()?
.cast_nullability(tgt_nullability, array.len(), ctx)?;

let result = match_each_integer_ptype!(tgt, |T| {
// Bit-packing rejects negative values, so sign extension of a packed value is the same as
// zero extension. A signed cast then gives the same bits as the unsigned cast of the same
// widths, and signed and unsigned casts share one set of kernels.
let unsigned_view;
let source = if src.is_signed_int() {
unsigned_view = unsigned_view_of(array, bit_width, ctx)?;
unsigned_view.as_view()
} else {
array
};

let result = match_each_unsigned_integer_ptype!(tgt.to_unsigned(), |T| {
let mut builder = PrimitiveBuilder::<T>::with_capacity_in(
tgt_nullability,
array.len(),
ctx.allocator(),
);
match_each_integer_ptype!(src, |F| {
unpack_map_into_builder::<F, T, _>(array, &mut builder, ctx, |v: F| v.as_())?;
match_each_unsigned_integer_ptype!(src.to_unsigned(), |F| {
// Only widening casts reach here. The const condition lets the compiler skip
// codegen of the other arms.
if const { F::PTYPE.byte_width() < T::PTYPE.byte_width() } {
unpack_map_into_builder::<F, T, _>(source, &mut builder, ctx, |v: F| v.as_())?;
} else {
unreachable!("{src} to {tgt} is not a widening integer cast");
}
});
builder.finish_into_primitive().into_array()
builder
.finish_into_primitive()
.reinterpret_cast(tgt)
.into_array()
});
Ok(Some(result))
}
Expand Down Expand Up @@ -144,6 +201,7 @@ mod tests {
use vortex_session::VortexSession;

use crate::BitPackedArray;
use crate::BitPackedArrayExt;
use crate::BitPackedData;

static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
Expand Down Expand Up @@ -176,6 +234,24 @@ mod tests {
);
}

/// Signed sources are cast through an unsigned view, which reinterprets the patch values.
#[rstest]
#[case::i8_to_i16(PType::I16)]
#[case::i8_to_i64(PType::I64)]
fn test_cast_bitpacked_signed_with_patches(#[case] tgt: PType) -> VortexResult<()> {
let values = PrimitiveArray::from_iter(
(0..2000).map(|i| if i % 97 == 0 { 120i8 } else { (i % 7) as i8 }),
);
let packed = bp(&values.clone().into_array(), 3);
assert!(packed.patches().is_some(), "test setup expects patches");

let target = DType::Primitive(tgt, Nullability::NonNullable);
let casted = packed.into_array().cast(target.clone())?;
let expected = values.into_array().cast(target)?;
assert_arrays_eq!(casted, expected, &mut SESSION.create_execution_ctx());
Ok(())
}

#[test]
fn test_cast_bitpacked_nullable() {
let values = PrimitiveArray::from_option_iter([Some(5u16), None, Some(10), Some(15), None]);
Expand Down
38 changes: 30 additions & 8 deletions vortex-array/src/aggregate_fn/fns/all_non_distinct/decimal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use vortex_error::VortexResult;
use crate::arrays::DecimalArray;
use crate::arrays::decimal::DecimalArrayExt;
use crate::dtype::BigCast;
use crate::dtype::NativeDecimalType;
use crate::match_each_decimal_value_type;

#[expect(
Expand All @@ -27,16 +28,37 @@ pub(super) fn check_decimal_identical(
match_each_decimal_value_type!(lhs.values_type(), |L| {
match_each_decimal_value_type!(rhs.values_type(), |R| {
match_each_decimal_value_type!(widest, |W| {
Ok(lhs
.buffer::<L>()
.iter()
.zip(rhs.buffer::<R>().iter())
.all(|(lhs, rhs)| {
<W as BigCast>::from(*lhs).vortex_expect("decimal widening should succeed")
== <W as BigCast>::from(*rhs)
// `L` and `R` differ and fix `W` as the wider of them, so only one arm can match.
// The const condition lets the compiler skip codegen of the other arms.
if const { is_widest_of_distinct::<L, R, W>() } {
Ok(lhs
.buffer::<L>()
.iter()
.zip(rhs.buffer::<R>().iter())
.all(|(lhs, rhs)| {
<W as BigCast>::from(*lhs)
.vortex_expect("decimal widening should succeed")
}))
== <W as BigCast>::from(*rhs)
.vortex_expect("decimal widening should succeed")
}))
} else {
unreachable!("{widest} is not the wider of two distinct decimal types")
}
})
})
})
}

/// Whether `L` and `R` differ and `W` is the wider of them.
const fn is_widest_of_distinct<L, R, W>() -> bool
where
L: NativeDecimalType,
R: NativeDecimalType,
W: NativeDecimalType,
{
let lhs = L::DECIMAL_TYPE as u8;
let rhs = R::DECIMAL_TYPE as u8;
let widest = if lhs > rhs { lhs } else { rhs };

lhs != rhs && W::DECIMAL_TYPE as u8 == widest
}
45 changes: 45 additions & 0 deletions vortex-array/src/aggregate_fn/fns/all_non_distinct/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
use std::sync::LazyLock;

use rstest::rstest;
use vortex_buffer::Buffer;
use vortex_buffer::buffer;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_session::VortexSession;

Expand All @@ -22,11 +24,14 @@ use crate::arrays::NullArray;
use crate::arrays::PrimitiveArray;
use crate::arrays::StructArray;
use crate::arrays::VarBinViewArray;
use crate::dtype::BigCast;
use crate::dtype::DType;
use crate::dtype::DecimalDType;
use crate::dtype::DecimalType;
use crate::dtype::FieldNames;
use crate::dtype::Nullability;
use crate::dtype::PType;
use crate::match_each_decimal_value_type;
use crate::validity::Validity;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
Expand Down Expand Up @@ -293,6 +298,46 @@ fn decimal_different_value_types() -> VortexResult<()> {
assert_matches_baseline(&a, &b)
}

/// Decimal values `[1, 2, last]` stored as `values_type`.
fn decimal_with_storage(values_type: DecimalType, last: i8) -> ArrayRef {
let dtype = DecimalDType::new(2, 0);
match_each_decimal_value_type!(values_type, |T| {
let values = [1i8, 2, last].map(|value| {
<T as BigCast>::from(value).vortex_expect("small values fit every decimal type")
});
DecimalArray::new(Buffer::copy_from(values), dtype, Validity::NonNullable).into_array()
})
}

/// Each pair of storage types selects a different widened comparison.
#[rstest]
fn decimal_every_storage_pair(
#[values(
DecimalType::I8,
DecimalType::I16,
DecimalType::I32,
DecimalType::I64,
DecimalType::I128,
DecimalType::I256
)]
lhs: DecimalType,
#[values(
DecimalType::I8,
DecimalType::I16,
DecimalType::I32,
DecimalType::I64,
DecimalType::I128,
DecimalType::I256
)]
rhs: DecimalType,
) -> VortexResult<()> {
assert_matches_baseline(&decimal_with_storage(lhs, 3), &decimal_with_storage(rhs, 3))?;
assert_matches_baseline(
&decimal_with_storage(lhs, 3),
&decimal_with_storage(rhs, -3),
)
}

#[test]
fn decimal_nullable() -> VortexResult<()> {
let dtype = DecimalDType::new(5, 2);
Expand Down
Loading
Loading