diff --git a/datafusion/functions/benches/datetime_expressions/date_part.rs b/datafusion/functions/benches/datetime_expressions/date_part.rs index 219f75f45e2ac..073a5fdd935da 100644 --- a/datafusion/functions/benches/datetime_expressions/date_part.rs +++ b/datafusion/functions/benches/datetime_expressions/date_part.rs @@ -188,6 +188,36 @@ fn generate_duration_nanosecond_array_with_nulls( ) } +fn generate_timestamp_ns_array_with_nulls(rng: &mut StdRng) -> TimestampNanosecondArray { + TimestampNanosecondArray::from( + (0..BATCH_SIZE) + .map(|_| { + if rng.random_bool(NULL_DENSITY) { + None + } else { + Some(rng.random_range(0..TS_BOUND * 1_000_000_000)) + } + }) + .collect::>(), + ) +} + +fn generate_time64_nanosecond_array_with_nulls( + rng: &mut StdRng, +) -> Time64NanosecondArray { + Time64NanosecondArray::from( + (0..BATCH_SIZE) + .map(|_| { + if rng.random_bool(NULL_DENSITY) { + None + } else { + Some(rng.random_range(0..SEC_DAY * 1_000_000_000)) + } + }) + .collect::>(), + ) +} + fn bench_date_part( c: &mut Criterion, udf: &Arc, @@ -244,6 +274,14 @@ fn criterion_benchmark(c: &mut Criterion) { let duration_ns_nulls = Arc::new(generate_duration_nanosecond_array_with_nulls(&mut rng)) as ArrayRef; + let ts_ns_tz = + Arc::new(generate_timestamp_ns_array(&mut rng).with_timezone("America/New_York")) + as ArrayRef; + let ts_ns_nulls = + Arc::new(generate_timestamp_ns_array_with_nulls(&mut rng)) as ArrayRef; + let time64_ns_nulls = + Arc::new(generate_time64_nanosecond_array_with_nulls(&mut rng)) as ArrayRef; + let udf = date_part(); for part in ["year", "month", "week", "day", "hour", "minute"] { @@ -365,6 +403,41 @@ fn criterion_benchmark(c: &mut Criterion) { ); } + // Durations past `i32::MAX` milliseconds or microseconds overflow `Int32`, so + // only `nanosecond` runs on them + for (part, return_type) in [ + ("millisecond", DataType::Int32), + ("microsecond", DataType::Int32), + ("nanosecond", DataType::Int64), + ] { + let mut arrays = vec![ + ("ns_tz", &ts_ns_tz), + ("ns_nulls", &ts_ns_nulls), + ("time32_ms", &time32_ms), + ("time64_ns", &time64_ns), + ("time64_ns_nulls", &time64_ns_nulls), + ("interval_dt", &interval_dt), + ("interval_mdn", &interval_mdn), + ("duration_ns_small", &duration_ns_small), + ]; + if part == "nanosecond" { + arrays.extend([ + ("duration_ns", &duration_ns), + ("duration_ns_nulls", &duration_ns_nulls), + ]); + } + for (name, array) in arrays { + bench_date_part( + c, + &udf, + &format!("date_part_{part}_{name}_1000"), + part, + Arc::clone(array), + return_type.clone(), + ); + } + } + for part in ["quarter", "isoyear", "doy", "dow", "isodow"] { bench_date_part( c, diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 6dcb948a0e514..919f1b0ac8b53 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -15,25 +15,33 @@ // specific language governing permissions and limitations // under the License. -use std::iter::repeat_n; use std::str::FromStr; use std::sync::Arc; -use arrow::array::temporal_conversions::as_datetime_with_timezone; +use arrow::array::temporal_conversions::{ + MICROSECONDS, MILLISECONDS, NANOSECONDS, SECONDS_IN_DAY, as_datetime_with_timezone, + timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_s_to_datetime, + timestamp_us_to_datetime, +}; use arrow::array::timezone::Tz; -use arrow::array::{Array, ArrayRef, Float64Array, Int32Array, Int64Array}; -use arrow::compute::{DatePart, binary, date_part}; +use arrow::array::{ + Array, ArrayRef, Float64Array, Int32Array, Int64Array, PrimitiveArray, +}; +use arrow::compute::{DatePart, cast, date_part}; use arrow::datatypes::DataType::{ Date32, Date64, Duration, Interval, Time32, Time64, Timestamp, }; use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; use arrow::datatypes::{ - ArrowTimestampType, DataType, Field, FieldRef, IntervalUnit as ArrowIntervalUnit, - TimeUnit, TimestampMicrosecondType, TimestampMillisecondType, - TimestampNanosecondType, TimestampSecondType, + ArrowPrimitiveType, ArrowTimestampType, DataType, Field, FieldRef, Int32Type, + Int64Type, IntervalUnit as ArrowIntervalUnit, Time32MillisecondType, + Time32SecondType, Time64MicrosecondType, Time64NanosecondType, + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, }; -use chrono::{Datelike, NaiveDate, Offset}; +use chrono::{Datelike, NaiveDate, NaiveDateTime, Offset, TimeZone, Timelike, Utc}; use datafusion_common::types::{NativeType, logical_date}; +use num_traits::AsPrimitive; use crate::datetime::common::parse_tz; use datafusion_common::{ @@ -41,14 +49,13 @@ use datafusion_common::{ cast::{ as_date32_array, as_date64_array, as_duration_microsecond_array, as_duration_millisecond_array, as_duration_nanosecond_array, - as_duration_second_array, as_int32_array, as_interval_dt_array, - as_interval_mdn_array, as_interval_ym_array, as_primitive_array, - as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array, - as_time64_nanosecond_array, as_timestamp_microsecond_array, - as_timestamp_millisecond_array, as_timestamp_nanosecond_array, - as_timestamp_second_array, + as_duration_second_array, as_interval_dt_array, as_interval_mdn_array, + as_interval_ym_array, as_primitive_array, as_time32_millisecond_array, + as_time32_second_array, as_time64_microsecond_array, as_time64_nanosecond_array, + as_timestamp_microsecond_array, as_timestamp_millisecond_array, + as_timestamp_nanosecond_array, as_timestamp_second_array, }, - exec_err, internal_err, not_impl_err, + exec_err, internal_err, types::logical_string, utils::take_function_args, }; @@ -191,13 +198,16 @@ impl ScalarUDFImpl for DatePartFunc { .flatten() .filter(|s| !s.is_empty()) .map(|part| { + // The integer parts are NULL for values arrow can't + // convert or that overflow, even when the input is + // not NULL if is_epoch(part) { Field::new(self.name(), DataType::Float64, nullable) } else if is_nanosecond(part) { - // See notes on [seconds_ns] for rationale - Field::new(self.name(), DataType::Int64, nullable) + // See notes on [subsecond_part] for rationale + Field::new(self.name(), DataType::Int64, true) } else { - Field::new(self.name(), DataType::Int32, nullable) + Field::new(self.name(), DataType::Int32, true) } }) }) @@ -232,10 +242,11 @@ impl ScalarUDFImpl for DatePartFunc { let part_trim = part_normalization(&part); let arr = match DatePart::from_str(part_trim) { - Ok(DatePart::Second) => seconds_as_i32(array.as_ref(), Second)?, - Ok(DatePart::Millisecond) => seconds_as_i32(array.as_ref(), Millisecond)?, - Ok(DatePart::Microsecond) => seconds_as_i32(array.as_ref(), Microsecond)?, - Ok(DatePart::Nanosecond) => seconds_ns(array.as_ref())?, + Ok( + part @ (DatePart::Millisecond + | DatePart::Microsecond + | DatePart::Nanosecond), + ) => subsecond_part(array.as_ref(), part)?, Ok(part) => date_part(array.as_ref(), part)?, Err(_) if is_epoch(part_trim) => epoch(array.as_ref())?, // `timezone`, `timezone_hour` and `timezone_minute` have no @@ -440,72 +451,6 @@ fn part_normalization(part: &str) -> &str { .unwrap_or(part) } -/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the -/// result to a total number of seconds, milliseconds, microseconds or -/// nanoseconds as an `Int32Array` -fn seconds_as_i32(array: &dyn Array, unit: TimeUnit) -> Result { - // Nanosecond is neither supported in Postgres nor DuckDB, to avoid dealing - // with overflow and precision issue we don't support nanosecond - if unit == Nanosecond { - return not_impl_err!("Date part {unit:?} not supported"); - } - - // Fast path with seconds - no need to compute nanoseconds - if unit == Second { - return Ok(date_part(array, DatePart::Second)?); - } - - // Fast path for Date32 and Date64 - no seconds - if array.data_type() == &Date32 || array.data_type() == &Date64 { - return Ok(Arc::new(Int32Array::from_iter_values_with_nulls( - repeat_n(0, array.len()), - array.nulls().cloned(), - ))); - } - - let conversion_factor = match unit { - Second => 1_000_000_000, - Millisecond => 1_000_000, - Microsecond => 1_000, - Nanosecond => 1, - }; - - let second_factor = match unit { - Second => 1, - Millisecond => 1_000, - Microsecond => 1_000_000, - Nanosecond => 1_000_000_000, - }; - - let secs = date_part(array, DatePart::Second)?; - // This assumes array is primitive and not a dictionary - let secs = as_int32_array(secs.as_ref())?; - let subsecs = date_part(array, DatePart::Nanosecond)?; - let subsecs = as_int32_array(subsecs.as_ref())?; - - // Special case where there are no nulls. - if subsecs.null_count() == 0 { - let r: Int32Array = binary(secs, subsecs, |secs, subsecs| { - secs * second_factor + (subsecs % 1_000_000_000) / conversion_factor - })?; - Ok(Arc::new(r)) - } else { - // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case - // where the number of nanoseconds overflows. - let r: Int32Array = secs - .iter() - .zip(subsecs) - .map(|(secs, subsecs)| { - secs.map(|secs| { - let subsecs = subsecs.unwrap_or(0); - secs * second_factor + (subsecs % 1_000_000_000) / conversion_factor - }) - }) - .collect(); - Ok(Arc::new(r)) - } -} - fn epoch(array: &dyn Array) -> Result { const SECONDS_IN_A_DAY: f64 = 86400_f64; // Note: Month-to-second conversion uses 30 days as an approximation. @@ -564,48 +509,189 @@ fn epoch(array: &dyn Array) -> Result { Ok(Arc::new(f)) } -/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the -/// result to a total number of nanoseconds as an Int64 array. +/// Return the seconds of `array`, with their fraction, as a number of +/// milliseconds, microseconds or nanoseconds (`part`): the seconds of the +/// minute for timestamps, times and intervals, and the total length for +/// durations. /// -/// This returns an Int64 rather than Int32 because there 1 billion -/// `nanosecond`s in each second, so representing up to 60 seconds as -/// nanoseconds can be values up to 60 billion, which does not fit in Int32. -fn seconds_ns(array: &dyn Array) -> Result { - // Fast path for Date32 and Date64 - no nanoseconds - if array.data_type() == &Date32 || array.data_type() == &Date64 { - return Ok(Arc::new(Int64Array::from_iter_values_with_nulls( - repeat_n(0, array.len()), - array.nulls().cloned(), - ))); - } +/// `nanosecond` returns Int64, because 60 seconds are 60 billion nanoseconds +/// and a duration can be up to `i64::MAX` nanoseconds long. `millisecond` and +/// `microsecond` return Int32. A duration whose length overflows the return +/// type is NULL. +fn subsecond_part(array: &dyn Array, part: DatePart) -> Result { + let len = array.len(); + let nulls = array.nulls().cloned(); + Ok(match (array.data_type(), part) { + // Fast path for Date32 and Date64 - no seconds + (Date32 | Date64, DatePart::Nanosecond) => { + Arc::new(Int64Array::new(vec![0; len].into(), nulls)) + } + (Date32 | Date64, _) => Arc::new(Int32Array::new(vec![0; len].into(), nulls)), + // arrow's `date_part(Nanosecond)` computes these, but returns Int32, + // which overflows past about 2.1 seconds + (Interval(ArrowIntervalUnit::DayTime), DatePart::Nanosecond) => Arc::new( + as_interval_dt_array(array)? + .unary::<_, Int64Type>(|x| (x.milliseconds % 60_000) as i64 * 1_000_000), + ), + (Interval(ArrowIntervalUnit::MonthDayNano), DatePart::Nanosecond) => Arc::new( + as_interval_mdn_array(array)? + .unary::<_, Int64Type>(|x| x.nanoseconds % 60_000_000_000), + ), + // arrow's cast returns NULL on overflow, like its `date_part`, and + // reinterprets `Duration(Nanosecond)` as Int64 without copying + (Duration(_), DatePart::Nanosecond) => { + cast(&cast(array, &Duration(Nanosecond))?, &DataType::Int64)? + } + // arrow's sub-second parts of intervals and durations already include + // the whole seconds, and it rejects them for `Interval(YearMonth)` + (Interval(_) | Duration(_), _) => date_part(array, part)?, + (_, DatePart::Millisecond) => { + seconds_with_fraction::(array, part)? + } + (_, DatePart::Microsecond) => { + seconds_with_fraction::(array, part)? + } + (_, DatePart::Nanosecond) => { + seconds_with_fraction::(array, part)? + } + (_, part) => return internal_err!("{part} is not a sub-second part"), + }) +} - let secs = date_part(array, DatePart::Second)?; - // This assumes array is primitive and not a dictionary - let secs = as_int32_array(secs.as_ref())?; - let subsecs = date_part(array, DatePart::Nanosecond)?; - let subsecs = as_int32_array(subsecs.as_ref())?; - - // Special case where there are no nulls. - if subsecs.null_count() == 0 { - let r: Int64Array = binary(secs, subsecs, |secs, subsecs| { - (secs as i64) * 1_000_000_000 + (subsecs as i64) - })?; - Ok(Arc::new(r)) - } else { - // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case - // where the number of nanoseconds overflows. - let r: Int64Array = secs - .iter() - .zip(subsecs) - .map(|(secs, subsecs)| { - secs.map(|secs| { - let subsecs = subsecs.unwrap_or(0); - (secs as i64) * 1_000_000_000 + (subsecs as i64) - }) +/// Return the seconds of the minute of timestamps (in their timezone) and +/// times, with their fraction, in units of `1 / PER_SECOND` seconds. +fn seconds_with_fraction( + array: &dyn Array, + part: DatePart, +) -> Result +where + O: ArrowPrimitiveType, + i64: AsPrimitive, +{ + Ok(match array.data_type() { + Timestamp(unit, tz) => { + let tz = parse_tz(tz.as_ref())?; + match unit { + Second => timestamp_seconds_with_fraction::< + TimestampSecondType, + O, + PER_SECOND, + >(array, tz, timestamp_s_to_datetime)?, + Millisecond => timestamp_seconds_with_fraction::< + TimestampMillisecondType, + O, + PER_SECOND, + >(array, tz, timestamp_ms_to_datetime)?, + Microsecond => timestamp_seconds_with_fraction::< + TimestampMicrosecondType, + O, + PER_SECOND, + >(array, tz, timestamp_us_to_datetime)?, + Nanosecond => timestamp_seconds_with_fraction::< + TimestampNanosecondType, + O, + PER_SECOND, + >(array, tz, timestamp_ns_to_datetime)?, + } + } + Time32(Second) => { + time_seconds_with_fraction::(array)? + } + Time32(Millisecond) => time_seconds_with_fraction::< + Time32MillisecondType, + O, + MILLISECONDS, + PER_SECOND, + >(array)?, + Time64(Microsecond) => time_seconds_with_fraction::< + Time64MicrosecondType, + O, + MICROSECONDS, + PER_SECOND, + >(array)?, + Time64(Nanosecond) => { + time_seconds_with_fraction::( + array, + )? + } + other => return internal_err!("{part} does not support {other}"), + }) +} + +fn timestamp_seconds_with_fraction( + array: &dyn Array, + tz: Option, + // arrow's converter for `T`, as its `date_part` uses. The generic + // `as_datetime::` measured about 1.5x slower. + to_datetime: impl Fn(i64) -> Option, +) -> Result +where + T: ArrowTimestampType, + O: ArrowPrimitiveType, + i64: AsPrimitive, +{ + let array = as_primitive_array::(array)?; + // One chrono conversion per row, where arrow's `date_part` needs one each + // for the seconds and the fraction. `unary_opt` keeps the input nulls and additionally nulls out any value + // that cannot be represented as a `DateTime` (out of chrono's range). + let r: PrimitiveArray = match tz { + Some(tz) => array.unary_opt(|v| { + to_datetime(v).map(|naive| { + seconds_of_minute::( + &Utc.from_utc_datetime(&naive).with_timezone(&tz), + ) }) - .collect(); - Ok(Arc::new(r)) - } + }), + None => array.unary_opt(|v| { + to_datetime(v).map(|naive| seconds_of_minute::(&naive)) + }), + }; + Ok(Arc::new(r)) +} + +/// The seconds of the minute of `t`, with their fraction, in units of +/// `1 / PER_SECOND` seconds +fn seconds_of_minute( + t: &impl Timelike, +) -> O::Native +where + i64: AsPrimitive, +{ + // At most 60 seconds, which fits in Int32 for milliseconds and + // microseconds, so the narrowing `as_` never truncates + (t.second() as i64 * PER_SECOND + t.nanosecond() as i64 / (NANOSECONDS / PER_SECOND)) + .as_() +} + +/// Like [`timestamp_seconds_with_fraction`] for times, which count +/// `1 / UNITS_PER_SECOND` seconds since midnight. +fn time_seconds_with_fraction( + array: &dyn Array, +) -> Result +where + T: ArrowPrimitiveType, + T::Native: AsPrimitive, + O: ArrowPrimitiveType, + i64: AsPrimitive, +{ + let array = as_primitive_array::(array)?; + // Values outside one day are NULL, as in arrow's `date_part`. `as_time` + // wraps some to a valid time: https://github.com/apache/arrow-rs/issues/11893 + let r: PrimitiveArray = array.unary_opt(|v| { + let v: i64 = v.as_(); + (0..SECONDS_IN_DAY * UNITS_PER_SECOND) + .contains(&v) + .then(|| { + let v = v % (60 * UNITS_PER_SECOND); + if PER_SECOND >= UNITS_PER_SECOND { + v * (PER_SECOND / UNITS_PER_SECOND) + } else { + v / (UNITS_PER_SECOND / PER_SECOND) + } + .as_() + }) + }); + Ok(Arc::new(r)) } #[cfg(test)] diff --git a/datafusion/spark/src/function/datetime/date_part.rs b/datafusion/spark/src/function/datetime/date_part.rs index 50ef1ece9d1a3..ed09ab08fbc8c 100644 --- a/datafusion/spark/src/function/datetime/date_part.rs +++ b/datafusion/spark/src/function/datetime/date_part.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::DataType; use datafusion_common::types::logical_date; use datafusion_common::{ Result, ScalarValue, internal_err, types::logical_string, utils::take_function_args, @@ -23,10 +23,9 @@ use datafusion_common::{ use datafusion_expr::expr::ScalarFunction; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ - Coercion, ColumnarValue, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, - Signature, TypeSignature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignature, TypeSignatureClass, Volatility, }; -use std::sync::Arc; /// Wrapper around datafusion date_part function to handle /// Spark behavior returning day of the week 1-indexed instead of 0-indexed and different part aliases. @@ -77,14 +76,11 @@ impl ScalarUDFImpl for SparkDatePart { &self.signature } + // The default `return_field_from_args` makes the result nullable, like the + // integer parts of `date_part`, which this simplifies to, because they can + // be NULL for a non-null input fn return_type(&self, _arg_types: &[DataType]) -> Result { - internal_err!("Use return_field_from_args in this case instead.") - } - - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + Ok(DataType::Int32) } fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index 96cbe78d49933..7fcfa77825241 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -912,6 +912,58 @@ SELECT extract(epoch from arrow_cast('23:32:50.123456789'::time, 'Time64(Nanosec ---- 84770.123456789 +# sub-second parts of times, with a NULL row +query IIII +SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t) +FROM (SELECT arrow_cast(column1, 'Time32(Second)') AS t + FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time))) +---- +50 50000 50000000 50000000000 +NULL NULL NULL NULL +1 1000 1000000 1000000000 + +query IIII +SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t) +FROM (SELECT arrow_cast(column1, 'Time32(Millisecond)') AS t + FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time))) +---- +50 50123 50123000 50123000000 +NULL NULL NULL NULL +1 1000 1000000 1000000000 + +query IIII +SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t) +FROM (SELECT arrow_cast(column1, 'Time64(Microsecond)') AS t + FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time))) +---- +50 50123 50123456 50123456000 +NULL NULL NULL NULL +1 1000 1000000 1000000000 + +query IIII +SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t) +FROM (SELECT arrow_cast(column1, 'Time64(Nanosecond)') AS t + FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time))) +---- +50 50123 50123456 50123456789 +NULL NULL NULL NULL +1 1000 1000000 1000000001 + +# times outside one day are NULL, like the other parts. The last row is 2^32 + 5 +# seconds, which must not wrap around to 5 seconds. +query IIII +SELECT extract(nanosecond from arrow_cast(arrow_cast(column1, 'Int32'), 'Time32(Second)')), + extract(nanosecond from arrow_cast(arrow_cast(column2, 'Int32'), 'Time32(Millisecond)')), + extract(nanosecond from arrow_cast(column3, 'Time64(Microsecond)')), + extract(nanosecond from arrow_cast(column4, 'Time64(Nanosecond)')) +FROM (VALUES (-1, -1, -1, -1), + (86400, 86400000, 86400000000, 86400000000000), + (NULL, NULL, 4294967301000000, 4294967301000000000)) +---- +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL + # test_extract_epoch query R @@ -974,6 +1026,70 @@ SELECT extract(epoch from arrow_cast('1969-12-31', 'Date64')) ---- -86400 +# sub-second parts of timestamps in each unit, before and after 1970, with a NULL row +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Second, None)') AS ts + FROM (VALUES ('2020-09-08T12:00:12'), (NULL), ('1969-12-31T23:59:58'))) +---- +12 12000 12000000 12000000000 +NULL NULL NULL NULL +58 58000 58000000 58000000000 + +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Millisecond, None)') AS ts + FROM (VALUES ('2020-09-08T12:00:12.123'), (NULL), ('1969-12-31T23:59:58.123'))) +---- +12 12123 12123000 12123000000 +NULL NULL NULL NULL +58 58123 58123000 58123000000 + +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Microsecond, None)') AS ts + FROM (VALUES ('2020-09-08T12:00:12.123456'), (NULL), ('1969-12-31T23:59:58.123456'))) +---- +12 12123 12123456 12123456000 +NULL NULL NULL NULL +58 58123 58123456 58123456000 + +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, None)') AS ts + FROM (VALUES ('2020-09-08T12:00:12.123456789'), (NULL), ('1969-12-31T23:59:58.123456789'))) +---- +12 12123 12123456 12123456789 +NULL NULL NULL NULL +58 58123 58123456 58123456789 + +# Africa/Monrovia was UTC-00:44:30 until 1972, so the seconds depend on the timezone +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Millisecond, Some("Africa/Monrovia"))') AS ts + FROM (VALUES (500), (-1500), (NULL))) +---- +30 30500 30500000 30500000000 +28 28500 28500000 28500000000 +NULL NULL NULL NULL + +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Africa/Monrovia"))') AS ts + FROM (VALUES (500000000), (-1500000000), (NULL))) +---- +30 30500 30500000 30500000000 +28 28500 28500000 28500000000 +NULL NULL NULL NULL + +# timestamps outside the range arrow represents are NULL, like the other parts +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)')), (NULL)) AS t(ts) +---- +NULL NULL NULL NULL +NULL NULL NULL NULL + # test_extract_interval query I @@ -1143,24 +1259,132 @@ create table t (id int, i interval) as values (4, interval '8 months'), (5, NULL); -query III +query IIII select id, extract(second from i), - extract(month from i) + extract(month from i), + extract(nanosecond from i) from t order by id; ---- -0 0 5 -1 0 3 -2 0 0 -3 2 0 -4 0 8 -5 NULL NULL +0 0 5 10 +1 0 3 0 +2 0 0 2000000 +3 2 0 2000000000 +4 0 8 0 +5 NULL NULL NULL statement ok drop table t; +# nanosecond on an interval is the seconds field (within the minute) in nanoseconds, +# like microsecond and millisecond. +# Reproducer from https://github.com/apache/datafusion/issues/26678 +query III +SELECT extract(nanosecond from interval '1.5 seconds'), + extract(microsecond from interval '1.5 seconds'), + extract(millisecond from interval '1.5 seconds') +---- +1500000000 1500000 1500 + +# past i32::MAX nanoseconds +query I +SELECT extract(nanosecond from interval '62.5 seconds') +---- +2500000000 + +query I +SELECT extract(nanosecond from interval '59.999999999 seconds') +---- +59999999999 + +query I +SELECT extract(nanosecond from interval '1 day 2 hours 3 minutes') +---- +0 + +query I +SELECT extract(nanosecond from interval '-1.5 seconds') +---- +-1500000000 + +query I +SELECT extract(nanosecond from arrow_cast('5.1 seconds', 'Interval(DayTime)')) +---- +5100000000 + +query I +SELECT extract(nanosecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')) +---- +2500000000 + +query I +SELECT extract(nanosecond from arrow_cast(NULL, 'Interval(DayTime)')) +---- +NULL + +# sub-second parts are the seconds field within the minute, with the sign of the +# milliseconds or nanoseconds field, whatever the days and months +query III +SELECT extract(millisecond from i), extract(microsecond from i), extract(nanosecond from i) +FROM (SELECT arrow_cast(column1, 'Interval(DayTime)') AS i FROM (VALUES + ('-62.5 seconds'), ('-60 seconds'), ('1 day 2 hours 5.1 seconds'), + ('-1 day 5.1 seconds'), ('-2 hours -5.1 seconds'), (NULL))) +---- +-2500 -2500000 -2500000000 +0 0 0 +5100 5100000 5100000000 +5100 5100000 5100000000 +-5100 -5100000 -5100000000 +NULL NULL NULL + +query III +SELECT extract(millisecond from i), extract(microsecond from i), extract(nanosecond from i) +FROM (SELECT arrow_cast(column1, 'Interval(MonthDayNano)') AS i FROM (VALUES + ('-62.5 seconds'), ('-60 seconds'), ('-59.999999999 seconds'), + ('1 month -62.5 seconds'), ('-1 day -62.5 seconds'), (NULL))) +---- +-2500 -2500000 -2500000000 +0 0 0 +-59999 -59999999 -59999999999 +-2500 -2500000 -2500000000 +-2500 -2500000 -2500000000 +NULL NULL NULL + +# millisecond and microsecond keep the fraction past i32::MAX nanoseconds. +# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638 +query IIII +SELECT extract(millisecond from interval '62.5 seconds'), + extract(microsecond from interval '62.5 seconds'), + extract(microsecond from interval '5.5 seconds'), + extract(microsecond from interval '59.999999 seconds') +---- +2500 2500000 5500000 59999999 + +query II +SELECT extract(millisecond from interval '-62.5 seconds'), + extract(microsecond from interval '-62.5 seconds') +---- +-2500 -2500000 + +query II +SELECT extract(millisecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')), + extract(microsecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')) +---- +2500 2500000 + +# Reproducer from https://github.com/apache/datafusion/issues/14738 +query III +SELECT date_part('microseconds', interval '1 hour'), + date_part('milliseconds', interval '1 hour'), + date_part('seconds', interval '1 hour') +---- +0 0 0 + +query error DataFusion error: Arrow error: Compute error: Nanosecond does not support: Interval\(YearMonth\) +SELECT extract(nanosecond from arrow_cast('10 months', 'Interval(YearMonth)')) + # test_extract_duration query I @@ -1289,6 +1513,146 @@ SELECT extract(day from arrow_cast(NULL, 'Duration(Second)')) ---- NULL +# nanosecond on a duration is its total length in nanoseconds. +# Reproducer from https://github.com/apache/datafusion/issues/26678 +query IIII +SELECT extract(nanosecond from arrow_cast(1500000000, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(1500000, 'Duration(Microsecond)')), + extract(nanosecond from arrow_cast(1500, 'Duration(Millisecond)')), + extract(nanosecond from arrow_cast(2, 'Duration(Second)')) +---- +1500000000 1500000000 1500000000 2000000000 + +# around i32::MAX nanoseconds +query III +SELECT extract(nanosecond from arrow_cast(2147483647, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(2147483648, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(62500000000, 'Duration(Nanosecond)')) +---- +2147483647 2147483648 62500000000 + +query II +SELECT extract(nanosecond from arrow_cast(-1500000000, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(-62500, 'Duration(Millisecond)')) +---- +-1500000000 -62500000000 + +query I +SELECT extract(nanosecond from arrow_cast(column1, 'Duration(Millisecond)')) +FROM (VALUES (1500), (NULL), (62500), (NULL), (-2)) +---- +1500000000 +NULL +62500000000 +NULL +-2000000 + +query II +SELECT extract(nanosecond from arrow_cast(9223372036854775807, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(-9223372036854775808, 'Duration(Nanosecond)')) +---- +9223372036854775807 -9223372036854775808 + +# values that overflow are NULL, like the other parts of a duration +query II +SELECT extract(nanosecond from arrow_cast(column1, 'Duration(Second)')), + extract(nanosecond from arrow_cast(column2, 'Duration(Millisecond)')) +FROM (VALUES (9223372036, -9223372036854), (9223372037, -9223372036855), (NULL, NULL)) +---- +9223372036000000000 -9223372036854000000 +NULL NULL +NULL NULL + +# millisecond and microsecond on a duration are its total length in that unit. +# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638 +query II +SELECT extract(millisecond from arrow_cast(2500000000, 'Duration(Nanosecond)')), + extract(microsecond from arrow_cast(2500000000, 'Duration(Nanosecond)')) +---- +2500 2500000 + +# truncates toward zero, like the other parts of a duration +query III +SELECT extract(millisecond from arrow_cast(-1500, 'Duration(Microsecond)')), + extract(millisecond from arrow_cast(-1500000, 'Duration(Nanosecond)')), + extract(second from arrow_cast(-1500, 'Duration(Millisecond)')) +---- +-1 -1 -1 + +query I +SELECT extract(microsecond from arrow_cast(column1, 'Duration(Nanosecond)')) +FROM (VALUES (5500000000), (NULL), (-62500000000), (NULL)) +---- +5500000 +NULL +-62500000 +NULL + +# around i32::MAX +query II +SELECT extract(millisecond from arrow_cast(2147483647, 'Duration(Millisecond)')), + extract(microsecond from arrow_cast(2147, 'Duration(Second)')) +---- +2147483647 2147000000 + +# values that overflow Int32 are NULL, like the other parts of a duration +query II +SELECT extract(millisecond from arrow_cast(column1, 'Duration(Millisecond)')), + extract(microsecond from arrow_cast(column2, 'Duration(Second)')) +FROM (VALUES (2147483648, 2148), (-2147483649, -2148), (NULL, NULL)) +---- +NULL NULL +NULL NULL +NULL NULL + +# Reproducer from https://github.com/apache/datafusion/issues/14738. Postgres returns 0, because +# there the difference is an interval. Here it is a Duration, whose microsecond is its total +# length, and 2.4e9 microseconds don't fit in Int32. +query I +SELECT date_part('microsecond', timestamp '1970-01-01T00:40:00' - timestamp '1970-01-01T00:00:00') +---- +NULL + +# Non-null inputs can give NULL results, so the result is nullable. +# Reproducers from https://github.com/apache/datafusion/issues/26685 +query I +SELECT extract(second from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts) +---- +NULL + +query I +SELECT extract(hour from d) FROM (VALUES (arrow_cast(9223372036854775807, 'Duration(Second)'))) AS t(d) +---- +NULL + +query I +SELECT extract(nanosecond from d) FROM (VALUES (arrow_cast(9223372037, 'Duration(Second)'))) AS t(d) +---- +NULL + +query I +SELECT extract(millisecond from d) FROM (VALUES (arrow_cast(2147483648, 'Duration(Millisecond)'))) AS t(d) +---- +NULL + +# epoch is NULL only for a NULL input, so it keeps the input's nullability +query TTT +DESCRIBE SELECT extract(epoch from ts) AS e, extract(second from ts) AS s, + extract(millisecond from ts) AS ms, extract(nanosecond from ts) AS ns +FROM (VALUES (timestamp '2020-01-01')) AS t(ts) +---- +e Float64 NO +s Int32 YES +ms Int32 YES +ns Int64 YES + +query TTT +DESCRIBE SELECT extract(epoch from ts) AS e, extract(second from ts) AS s +FROM (VALUES (timestamp '2020-01-01'), (NULL)) AS t(ts) +---- +e Float64 YES +s Int32 YES + # test_extract_date_part_func query B diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt index beece06d66bbf..566179ba14f0c 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt @@ -284,3 +284,10 @@ FROM VALUES 1999 2000 NULL + +# Nullable even for a non-null input, like the integer parts of the date_part +# this simplifies to +query TTT +DESCRIBE SELECT date_part('SECOND'::string, ts) AS s FROM (VALUES (timestamp '2020-01-01')) AS t(ts) +---- +s Int32 YES