From c920a5b92df0172b23026c3773dc7cd0fbdb8ce6 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Fri, 23 Oct 2026 16:49:59 -0400 Subject: [PATCH 01/11] bench: add date_part millisecond, microsecond and nanosecond cases for times, intervals, durations and timezones --- .../benches/datetime_expressions/date_part.rs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/datafusion/functions/benches/datetime_expressions/date_part.rs b/datafusion/functions/benches/datetime_expressions/date_part.rs index 219f75f45e2ac..48f9742dc763e 100644 --- a/datafusion/functions/benches/datetime_expressions/date_part.rs +++ b/datafusion/functions/benches/datetime_expressions/date_part.rs @@ -244,6 +244,10 @@ fn criterion_benchmark(c: &mut Criterion) { let duration_ns_nulls = Arc::new(generate_duration_nanosecond_array_with_nulls(&mut rng)) as ArrayRef; + let ts_ns_tz = + Arc::new(generate_timestamp_ns_array(&mut rng).with_timezone("America/New_York")) + as ArrayRef; + let udf = date_part(); for part in ["year", "month", "week", "day", "hour", "minute"] { @@ -365,6 +369,39 @@ fn criterion_benchmark(c: &mut Criterion) { ); } + // Durations past `i32::MAX` milliseconds or microseconds overflow `Int32`, so + // only `nanosecond` runs on them + for (part, return_type) in [ + ("millisecond", DataType::Int32), + ("microsecond", DataType::Int32), + ("nanosecond", DataType::Int64), + ] { + let mut arrays = vec![ + ("ns_tz", &ts_ns_tz), + ("time32_ms", &time32_ms), + ("time64_ns", &time64_ns), + ("interval_dt", &interval_dt), + ("interval_mdn", &interval_mdn), + ("duration_ns_small", &duration_ns_small), + ]; + if part == "nanosecond" { + arrays.extend([ + ("duration_ns", &duration_ns), + ("duration_ns_nulls", &duration_ns_nulls), + ]); + } + for (name, array) in arrays { + bench_date_part( + c, + &udf, + &format!("date_part_{part}_{name}_1000"), + part, + Arc::clone(array), + return_type.clone(), + ); + } + } + for part in ["quarter", "isoyear", "doy", "dow", "isodow"] { bench_date_part( c, From 1d4f93b17d392e8d481f628fb08dd231f0bc21c7 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Fri, 23 Oct 2026 16:50:08 -0400 Subject: [PATCH 02/11] fix: keep fractional seconds in EXTRACT(MILLISECOND/MICROSECOND/NANOSECOND) for Duration and Interval --- .../functions/src/datetime/date_part.rs | 193 +++++++-------- .../test_files/datetime/date_part.slt | 227 +++++++++++++++++- 2 files changed, 302 insertions(+), 118 deletions(-) diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 6dcb948a0e514..eadabcdbd87e5 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -15,25 +15,27 @@ // specific language governing permissions and limitations // under the License. -use std::iter::repeat_n; use std::str::FromStr; use std::sync::Arc; use arrow::array::temporal_conversions::as_datetime_with_timezone; use arrow::array::timezone::Tz; -use arrow::array::{Array, ArrayRef, Float64Array, Int32Array, Int64Array}; -use arrow::compute::{DatePart, binary, date_part}; +use arrow::array::{ + Array, ArrayRef, Float64Array, Int32Array, Int64Array, PrimitiveArray, +}; +use arrow::compute::{DatePart, binary, cast, date_part}; use arrow::datatypes::DataType::{ Date32, Date64, Duration, Interval, Time32, Time64, Timestamp, }; use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; use arrow::datatypes::{ - ArrowTimestampType, DataType, Field, FieldRef, IntervalUnit as ArrowIntervalUnit, - TimeUnit, TimestampMicrosecondType, TimestampMillisecondType, - TimestampNanosecondType, TimestampSecondType, + ArrowPrimitiveType, ArrowTimestampType, DataType, Field, FieldRef, Int32Type, + Int64Type, IntervalUnit as ArrowIntervalUnit, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; use chrono::{Datelike, NaiveDate, Offset}; use datafusion_common::types::{NativeType, logical_date}; +use num_traits::AsPrimitive; use crate::datetime::common::parse_tz; use datafusion_common::{ @@ -48,7 +50,7 @@ use datafusion_common::{ as_timestamp_millisecond_array, as_timestamp_nanosecond_array, as_timestamp_second_array, }, - exec_err, internal_err, not_impl_err, + exec_err, internal_err, types::logical_string, utils::take_function_args, }; @@ -232,9 +234,12 @@ impl ScalarUDFImpl for DatePartFunc { let part_trim = part_normalization(&part); let arr = match DatePart::from_str(part_trim) { - Ok(DatePart::Second) => seconds_as_i32(array.as_ref(), Second)?, - Ok(DatePart::Millisecond) => seconds_as_i32(array.as_ref(), Millisecond)?, - Ok(DatePart::Microsecond) => seconds_as_i32(array.as_ref(), Microsecond)?, + Ok(part @ DatePart::Millisecond) => { + seconds_as_i32::<1_000>(array.as_ref(), part)? + } + Ok(part @ DatePart::Microsecond) => { + seconds_as_i32::<1_000_000>(array.as_ref(), part)? + } Ok(DatePart::Nanosecond) => seconds_ns(array.as_ref())?, Ok(part) => date_part(array.as_ref(), part)?, Err(_) if is_epoch(part_trim) => epoch(array.as_ref())?, @@ -440,72 +445,6 @@ fn part_normalization(part: &str) -> &str { .unwrap_or(part) } -/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the -/// result to a total number of seconds, milliseconds, microseconds or -/// nanoseconds as an `Int32Array` -fn seconds_as_i32(array: &dyn Array, unit: TimeUnit) -> Result { - // Nanosecond is neither supported in Postgres nor DuckDB, to avoid dealing - // with overflow and precision issue we don't support nanosecond - if unit == Nanosecond { - return not_impl_err!("Date part {unit:?} not supported"); - } - - // Fast path with seconds - no need to compute nanoseconds - if unit == Second { - return Ok(date_part(array, DatePart::Second)?); - } - - // Fast path for Date32 and Date64 - no seconds - if array.data_type() == &Date32 || array.data_type() == &Date64 { - return Ok(Arc::new(Int32Array::from_iter_values_with_nulls( - repeat_n(0, array.len()), - array.nulls().cloned(), - ))); - } - - let conversion_factor = match unit { - Second => 1_000_000_000, - Millisecond => 1_000_000, - Microsecond => 1_000, - Nanosecond => 1, - }; - - let second_factor = match unit { - Second => 1, - Millisecond => 1_000, - Microsecond => 1_000_000, - Nanosecond => 1_000_000_000, - }; - - let secs = date_part(array, DatePart::Second)?; - // This assumes array is primitive and not a dictionary - let secs = as_int32_array(secs.as_ref())?; - let subsecs = date_part(array, DatePart::Nanosecond)?; - let subsecs = as_int32_array(subsecs.as_ref())?; - - // Special case where there are no nulls. - if subsecs.null_count() == 0 { - let r: Int32Array = binary(secs, subsecs, |secs, subsecs| { - secs * second_factor + (subsecs % 1_000_000_000) / conversion_factor - })?; - Ok(Arc::new(r)) - } else { - // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case - // where the number of nanoseconds overflows. - let r: Int32Array = secs - .iter() - .zip(subsecs) - .map(|(secs, subsecs)| { - secs.map(|secs| { - let subsecs = subsecs.unwrap_or(0); - secs * second_factor + (subsecs % 1_000_000_000) / conversion_factor - }) - }) - .collect(); - Ok(Arc::new(r)) - } -} - fn epoch(array: &dyn Array) -> Result { const SECONDS_IN_A_DAY: f64 = 86400_f64; // Note: Month-to-second conversion uses 30 days as an approximation. @@ -564,48 +503,82 @@ fn epoch(array: &dyn Array) -> Result { Ok(Arc::new(f)) } -/// Invoke [`date_part`] on an `array` (e.g. Timestamp) and convert the -/// result to a total number of nanoseconds as an Int64 array. +/// Return the seconds of `array`, with their fraction, as a number of +/// milliseconds or microseconds (`part`), as an Int32 array. +fn seconds_as_i32( + array: &dyn Array, + part: DatePart, +) -> Result { + match array.data_type() { + // Fast path for Date32 and Date64 - no seconds + Date32 | Date64 => Ok(Arc::new(Int32Array::new( + vec![0; array.len()].into(), + array.nulls().cloned(), + ))), + // arrow's sub-second parts of intervals and durations already include + // the whole seconds + Interval(ArrowIntervalUnit::DayTime | ArrowIntervalUnit::MonthDayNano) + | Duration(_) => Ok(date_part(array, part)?), + _ => seconds_with_fraction::(array, part), + } +} + +/// Return the seconds of `array`, with their fraction, as a number of +/// nanoseconds: the seconds of the minute for timestamps, times and intervals, +/// and the total length for durations. /// -/// This returns an Int64 rather than Int32 because there 1 billion +/// This returns an Int64 rather than Int32 because there are 1 billion /// `nanosecond`s in each second, so representing up to 60 seconds as /// nanoseconds can be values up to 60 billion, which does not fit in Int32. fn seconds_ns(array: &dyn Array) -> Result { - // Fast path for Date32 and Date64 - no nanoseconds - if array.data_type() == &Date32 || array.data_type() == &Date64 { - return Ok(Arc::new(Int64Array::from_iter_values_with_nulls( - repeat_n(0, array.len()), - array.nulls().cloned(), - ))); - } + let r: Int64Array = match array.data_type() { + // Fast path for Date32 and Date64 - no nanoseconds + Date32 | Date64 => { + Int64Array::new(vec![0; array.len()].into(), array.nulls().cloned()) + } + // arrow's `date_part(Nanosecond)` computes these, but returns Int32, + // which overflows past about 2.1 seconds + Interval(ArrowIntervalUnit::DayTime) => as_interval_dt_array(array)? + .unary(|x| (x.milliseconds % 60_000) as i64 * 1_000_000), + Interval(ArrowIntervalUnit::MonthDayNano) => { + as_interval_mdn_array(array)?.unary(|x| x.nanoseconds % 60_000_000_000) + } + Duration(Nanosecond) => as_duration_nanosecond_array(array)?.reinterpret_cast(), + // arrow's cast returns NULL on overflow, like its `date_part` + Duration(_) => { + as_duration_nanosecond_array(&cast(array, &Duration(Nanosecond))?)? + .reinterpret_cast() + } + _ => { + return seconds_with_fraction::( + array, + DatePart::Nanosecond, + ); + } + }; + Ok(Arc::new(r)) +} +/// Combine arrow's seconds of the minute with its fraction of the second in +/// `part` units, for types where arrow returns them separately (timestamps +/// and times). +fn seconds_with_fraction( + array: &dyn Array, + part: DatePart, +) -> Result +where + O: ArrowPrimitiveType, + i64: AsPrimitive, +{ let secs = date_part(array, DatePart::Second)?; // This assumes array is primitive and not a dictionary let secs = as_int32_array(secs.as_ref())?; - let subsecs = date_part(array, DatePart::Nanosecond)?; - let subsecs = as_int32_array(subsecs.as_ref())?; - - // Special case where there are no nulls. - if subsecs.null_count() == 0 { - let r: Int64Array = binary(secs, subsecs, |secs, subsecs| { - (secs as i64) * 1_000_000_000 + (subsecs as i64) - })?; - Ok(Arc::new(r)) - } else { - // Nulls in secs are preserved, nulls in subsecs are treated as zero to account for the case - // where the number of nanoseconds overflows. - let r: Int64Array = secs - .iter() - .zip(subsecs) - .map(|(secs, subsecs)| { - secs.map(|secs| { - let subsecs = subsecs.unwrap_or(0); - (secs as i64) * 1_000_000_000 + (subsecs as i64) - }) - }) - .collect(); - Ok(Arc::new(r)) - } + let fraction = date_part(array, part)?; + let fraction = as_int32_array(fraction.as_ref())?; + let r: PrimitiveArray = binary(secs, fraction, |secs, fraction| { + (secs as i64 * PER_SECOND + fraction as i64).as_() + })?; + Ok(Arc::new(r)) } #[cfg(test)] diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index 96cbe78d49933..9c87866fbca69 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -974,6 +974,40 @@ SELECT extract(epoch from arrow_cast('1969-12-31', 'Date64')) ---- -86400 +# sub-second parts of timestamps before 1970 +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (VALUES + (timestamp '1969-12-31T23:59:58.123456789'), + (arrow_cast('1969-12-31T23:59:58.123', 'Timestamp(Millisecond, None)')), + (arrow_cast('1969-12-31T23:59:58', 'Timestamp(Second, None)')) +) AS t(ts) +---- +58 58123 58123456 58123456789 +58 58123 58123000 58123000000 +58 58000 58000000 58000000000 + +# Africa/Monrovia was UTC-00:44:30 until 1972, so the seconds depend on the timezone +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (VALUES + (arrow_cast(500, 'Timestamp(Millisecond, Some("Africa/Monrovia"))')), + (arrow_cast(-1500, 'Timestamp(Millisecond, Some("Africa/Monrovia"))')), + (arrow_cast(500000000, 'Timestamp(Nanosecond, Some("Africa/Monrovia"))')) +) AS t(ts) +---- +30 30500 30500000 30500000000 +28 28500 28500000 28500000000 +30 30500 30500000 30500000000 + +# timestamps outside the range arrow represents are NULL, like the other parts +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)')), (NULL)) AS t(ts) +---- +NULL NULL NULL NULL +NULL NULL NULL NULL + # test_extract_interval query I @@ -1143,24 +1177,105 @@ create table t (id int, i interval) as values (4, interval '8 months'), (5, NULL); -query III +query IIII select id, extract(second from i), - extract(month from i) + extract(month from i), + extract(nanosecond from i) from t order by id; ---- -0 0 5 -1 0 3 -2 0 0 -3 2 0 -4 0 8 -5 NULL NULL +0 0 5 10 +1 0 3 0 +2 0 0 2000000 +3 2 0 2000000000 +4 0 8 0 +5 NULL NULL NULL statement ok drop table t; +# nanosecond on an interval is the seconds field (within the minute) in nanoseconds, +# like microsecond and millisecond +query III +SELECT extract(nanosecond from interval '1.5 seconds'), + extract(microsecond from interval '1.5 seconds'), + extract(millisecond from interval '1.5 seconds') +---- +1500000000 1500000 1500 + +# past i32::MAX nanoseconds +query I +SELECT extract(nanosecond from interval '62.5 seconds') +---- +2500000000 + +query I +SELECT extract(nanosecond from interval '59.999999999 seconds') +---- +59999999999 + +query I +SELECT extract(nanosecond from interval '1 day 2 hours 3 minutes') +---- +0 + +query I +SELECT extract(nanosecond from interval '-1.5 seconds') +---- +-1500000000 + +query I +SELECT extract(nanosecond from arrow_cast('5.1 seconds', 'Interval(DayTime)')) +---- +5100000000 + +query I +SELECT extract(nanosecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')) +---- +2500000000 + +query I +SELECT extract(nanosecond from arrow_cast(NULL, 'Interval(DayTime)')) +---- +NULL + +# millisecond and microsecond keep the fraction past i32::MAX nanoseconds +query IIII +SELECT extract(millisecond from interval '62.5 seconds'), + extract(microsecond from interval '62.5 seconds'), + extract(microsecond from interval '5.5 seconds'), + extract(microsecond from interval '59.999999 seconds') +---- +2500 2500000 5500000 59999999 + +query II +SELECT extract(millisecond from interval '-62.5 seconds'), + extract(microsecond from interval '-62.5 seconds') +---- +-2500 -2500000 + +query II +SELECT extract(millisecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')), + extract(microsecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')) +---- +2500 2500000 + +# Reproducer from https://github.com/apache/datafusion/issues/14738 +query III +SELECT date_part('microseconds', interval '1 hour'), + date_part('milliseconds', interval '1 hour'), + date_part('seconds', interval '1 hour') +---- +0 0 0 + +query error DataFusion error: Arrow error: Compute error: Second does not support: Interval\(YearMonth\) +SELECT extract(nanosecond from arrow_cast('10 months', 'Interval(YearMonth)')) + +query error DataFusion error: Arrow error: Compute error: Second does not support: Interval\(YearMonth\) +SELECT extract(millisecond from arrow_cast('10 months', 'Interval(YearMonth)')) + # test_extract_duration query I @@ -1289,6 +1404,102 @@ SELECT extract(day from arrow_cast(NULL, 'Duration(Second)')) ---- NULL +# nanosecond on a duration is its total length in nanoseconds +query IIII +SELECT extract(nanosecond from arrow_cast(1500000000, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(1500000, 'Duration(Microsecond)')), + extract(nanosecond from arrow_cast(1500, 'Duration(Millisecond)')), + extract(nanosecond from arrow_cast(2, 'Duration(Second)')) +---- +1500000000 1500000000 1500000000 2000000000 + +# around i32::MAX nanoseconds +query III +SELECT extract(nanosecond from arrow_cast(2147483647, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(2147483648, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(62500000000, 'Duration(Nanosecond)')) +---- +2147483647 2147483648 62500000000 + +query II +SELECT extract(nanosecond from arrow_cast(-1500000000, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(-62500, 'Duration(Millisecond)')) +---- +-1500000000 -62500000000 + +query I +SELECT extract(nanosecond from arrow_cast(column1, 'Duration(Millisecond)')) +FROM (VALUES (1500), (NULL), (62500), (NULL), (-2)) +---- +1500000000 +NULL +62500000000 +NULL +-2000000 + +query II +SELECT extract(nanosecond from arrow_cast(9223372036854775807, 'Duration(Nanosecond)')), + extract(nanosecond from arrow_cast(-9223372036854775808, 'Duration(Nanosecond)')) +---- +9223372036854775807 -9223372036854775808 + +# values that overflow are NULL, like the other parts of a duration +query II +SELECT extract(nanosecond from arrow_cast(column1, 'Duration(Second)')), + extract(nanosecond from arrow_cast(column2, 'Duration(Millisecond)')) +FROM (VALUES (9223372036, -9223372036854), (9223372037, -9223372036855), (NULL, NULL)) +---- +9223372036000000000 -9223372036854000000 +NULL NULL +NULL NULL + +# millisecond and microsecond on a duration are its total length in that unit +query II +SELECT extract(millisecond from arrow_cast(5500000000, 'Duration(Nanosecond)')), + extract(microsecond from arrow_cast(5500000000, 'Duration(Nanosecond)')) +---- +5500 5500000 + +# truncates toward zero, like the other parts of a duration +query III +SELECT extract(millisecond from arrow_cast(-1500, 'Duration(Microsecond)')), + extract(millisecond from arrow_cast(-1500000, 'Duration(Nanosecond)')), + extract(second from arrow_cast(-1500, 'Duration(Millisecond)')) +---- +-1 -1 -1 + +query I +SELECT extract(microsecond from arrow_cast(column1, 'Duration(Nanosecond)')) +FROM (VALUES (5500000000), (NULL), (-62500000000), (NULL)) +---- +5500000 +NULL +-62500000 +NULL + +# around i32::MAX +query II +SELECT extract(millisecond from arrow_cast(2147483647, 'Duration(Millisecond)')), + extract(microsecond from arrow_cast(2147, 'Duration(Second)')) +---- +2147483647 2147000000 + +# values that overflow Int32 are NULL, like the other parts of a duration +query II +SELECT extract(millisecond from arrow_cast(column1, 'Duration(Millisecond)')), + extract(microsecond from arrow_cast(column2, 'Duration(Second)')) +FROM (VALUES (2147483648, 2148), (-2147483649, -2148), (NULL, NULL)) +---- +NULL NULL +NULL NULL +NULL NULL + +# Reproducer from https://github.com/apache/datafusion/issues/14738: 2.4e9 microseconds don't fit in Int32 +query I +SELECT date_part('microsecond', timestamp '1970-01-01T00:40:00' - timestamp '1970-01-01T00:00:00') +---- +NULL + # test_extract_date_part_func query B From 5bf4908e80436bd973ee9a2301e6f3e32a036f15 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Fri, 23 Oct 2026 17:21:22 -0400 Subject: [PATCH 03/11] fix: merge date_part sub-second helpers and make integer results nullable again --- .../functions/src/datetime/date_part.rs | 104 +++++++++--------- .../test_files/datetime/date_part.slt | 21 ++++ 2 files changed, 73 insertions(+), 52 deletions(-) diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index eadabcdbd87e5..6871fdb7927c0 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -193,13 +193,16 @@ impl ScalarUDFImpl for DatePartFunc { .flatten() .filter(|s| !s.is_empty()) .map(|part| { + // The integer parts are NULL for values arrow can't + // convert or that overflow, even when the input is + // not NULL if is_epoch(part) { Field::new(self.name(), DataType::Float64, nullable) } else if is_nanosecond(part) { - // See notes on [seconds_ns] for rationale - Field::new(self.name(), DataType::Int64, nullable) + // See notes on [subsecond_part] for rationale + Field::new(self.name(), DataType::Int64, true) } else { - Field::new(self.name(), DataType::Int32, nullable) + Field::new(self.name(), DataType::Int32, true) } }) }) @@ -234,13 +237,11 @@ impl ScalarUDFImpl for DatePartFunc { let part_trim = part_normalization(&part); let arr = match DatePart::from_str(part_trim) { - Ok(part @ DatePart::Millisecond) => { - seconds_as_i32::<1_000>(array.as_ref(), part)? - } - Ok(part @ DatePart::Microsecond) => { - seconds_as_i32::<1_000_000>(array.as_ref(), part)? - } - Ok(DatePart::Nanosecond) => seconds_ns(array.as_ref())?, + Ok( + part @ (DatePart::Millisecond + | DatePart::Microsecond + | DatePart::Nanosecond), + ) => subsecond_part(array.as_ref(), part)?, Ok(part) => date_part(array.as_ref(), part)?, Err(_) if is_epoch(part_trim) => epoch(array.as_ref())?, // `timezone`, `timezone_hour` and `timezone_minute` have no @@ -504,59 +505,58 @@ fn epoch(array: &dyn Array) -> Result { } /// Return the seconds of `array`, with their fraction, as a number of -/// milliseconds or microseconds (`part`), as an Int32 array. -fn seconds_as_i32( - array: &dyn Array, - part: DatePart, -) -> Result { - match array.data_type() { - // Fast path for Date32 and Date64 - no seconds - Date32 | Date64 => Ok(Arc::new(Int32Array::new( - vec![0; array.len()].into(), - array.nulls().cloned(), - ))), - // arrow's sub-second parts of intervals and durations already include - // the whole seconds - Interval(ArrowIntervalUnit::DayTime | ArrowIntervalUnit::MonthDayNano) - | Duration(_) => Ok(date_part(array, part)?), - _ => seconds_with_fraction::(array, part), - } -} - -/// Return the seconds of `array`, with their fraction, as a number of -/// nanoseconds: the seconds of the minute for timestamps, times and intervals, -/// and the total length for durations. +/// milliseconds, microseconds or nanoseconds (`part`): the seconds of the +/// minute for timestamps, times and intervals, and the total length for +/// durations. /// -/// This returns an Int64 rather than Int32 because there are 1 billion +/// `nanosecond` returns an Int64 rather than Int32 because there are 1 billion /// `nanosecond`s in each second, so representing up to 60 seconds as /// nanoseconds can be values up to 60 billion, which does not fit in Int32. -fn seconds_ns(array: &dyn Array) -> Result { - let r: Int64Array = match array.data_type() { - // Fast path for Date32 and Date64 - no nanoseconds - Date32 | Date64 => { - Int64Array::new(vec![0; array.len()].into(), array.nulls().cloned()) +fn subsecond_part(array: &dyn Array, part: DatePart) -> Result { + let len = array.len(); + let nulls = array.nulls().cloned(); + Ok(match (array.data_type(), part) { + // Fast path for Date32 and Date64 - no seconds + (Date32 | Date64, DatePart::Nanosecond) => { + Arc::new(Int64Array::new(vec![0; len].into(), nulls)) } + (Date32 | Date64, _) => Arc::new(Int32Array::new(vec![0; len].into(), nulls)), // arrow's `date_part(Nanosecond)` computes these, but returns Int32, // which overflows past about 2.1 seconds - Interval(ArrowIntervalUnit::DayTime) => as_interval_dt_array(array)? - .unary(|x| (x.milliseconds % 60_000) as i64 * 1_000_000), - Interval(ArrowIntervalUnit::MonthDayNano) => { - as_interval_mdn_array(array)?.unary(|x| x.nanoseconds % 60_000_000_000) + (Interval(ArrowIntervalUnit::DayTime), DatePart::Nanosecond) => Arc::new( + as_interval_dt_array(array)? + .unary::<_, Int64Type>(|x| (x.milliseconds % 60_000) as i64 * 1_000_000), + ), + (Interval(ArrowIntervalUnit::MonthDayNano), DatePart::Nanosecond) => Arc::new( + as_interval_mdn_array(array)? + .unary::<_, Int64Type>(|x| x.nanoseconds % 60_000_000_000), + ), + (Duration(Nanosecond), DatePart::Nanosecond) => { + Arc::new(as_duration_nanosecond_array(array)?.reinterpret_cast::()) } - Duration(Nanosecond) => as_duration_nanosecond_array(array)?.reinterpret_cast(), // arrow's cast returns NULL on overflow, like its `date_part` - Duration(_) => { + (Duration(_), DatePart::Nanosecond) => Arc::new( as_duration_nanosecond_array(&cast(array, &Duration(Nanosecond))?)? - .reinterpret_cast() + .reinterpret_cast::(), + ), + // arrow's sub-second parts of intervals and durations already include + // the whole seconds + ( + Interval(ArrowIntervalUnit::DayTime | ArrowIntervalUnit::MonthDayNano) + | Duration(_), + _, + ) => date_part(array, part)?, + (_, DatePart::Millisecond) => { + seconds_with_fraction::(array, part)? } - _ => { - return seconds_with_fraction::( - array, - DatePart::Nanosecond, - ); + (_, DatePart::Microsecond) => { + seconds_with_fraction::(array, part)? } - }; - Ok(Arc::new(r)) + (_, DatePart::Nanosecond) => { + seconds_with_fraction::(array, part)? + } + (_, part) => return internal_err!("{part} is not a sub-second part"), + }) } /// Combine arrow's seconds of the minute with its fraction of the second in diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index 9c87866fbca69..f0137d48d3cdd 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -1500,6 +1500,27 @@ SELECT date_part('microsecond', timestamp '1970-01-01T00:40:00' - timestamp '197 ---- NULL +# Non-null inputs can give NULL results, so the result is nullable +query I +SELECT extract(second from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts) +---- +NULL + +query I +SELECT extract(hour from d) FROM (VALUES (arrow_cast(9223372036854775807, 'Duration(Second)'))) AS t(d) +---- +NULL + +query I +SELECT extract(nanosecond from d) FROM (VALUES (arrow_cast(9223372037, 'Duration(Second)'))) AS t(d) +---- +NULL + +query I +SELECT extract(millisecond from d) FROM (VALUES (arrow_cast(2147483648, 'Duration(Millisecond)'))) AS t(d) +---- +NULL + # test_extract_date_part_func query B From a6e86f069200b5158c3fd7d0b1109979e7be243a Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Fri, 23 Oct 2026 17:31:44 -0400 Subject: [PATCH 04/11] fix: make Spark date_part result nullable to match date_part --- .../spark/src/function/datetime/date_part.rs | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/datafusion/spark/src/function/datetime/date_part.rs b/datafusion/spark/src/function/datetime/date_part.rs index 50ef1ece9d1a3..8946ee90aad4e 100644 --- a/datafusion/spark/src/function/datetime/date_part.rs +++ b/datafusion/spark/src/function/datetime/date_part.rs @@ -81,10 +81,10 @@ impl ScalarUDFImpl for SparkDatePart { internal_err!("Use return_field_from_args in this case instead.") } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); - - Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { + // Nullable like the integer parts of `date_part`, which this simplifies + // to, because they can be NULL for a non-null input + Ok(Arc::new(Field::new(self.name(), DataType::Int32, true))) } fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { @@ -127,3 +127,39 @@ impl ScalarUDFImpl for SparkDatePart { Ok(ExprSimplifyResult::Simplified(date_part_expr)) } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow::datatypes::TimeUnit; + + /// The result field matches the field of the `date_part` it simplifies to. + /// That `date_part` can return NULL for a non-null input, so both are + /// nullable. + #[test] + fn test_date_part_nullability_matches_simplified() -> Result<()> { + let arg_fields: Vec = vec![ + Arc::new(Field::new("part", DataType::Utf8, false)), + Arc::new(Field::new( + "ts", + DataType::Timestamp(TimeUnit::Second, None), + false, + )), + ]; + let part = ScalarValue::from("second"); + let scalar_arguments = [Some(&part), None]; + let args = || ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &scalar_arguments, + }; + + let spark = SparkDatePart::new().return_field_from_args(args())?; + let simplified = + datafusion_functions::datetime::date_part().return_field_from_args(args())?; + assert!(spark.is_nullable()); + assert_eq!(spark.is_nullable(), simplified.is_nullable()); + assert_eq!(spark.data_type(), simplified.data_type()); + + Ok(()) + } +} From adab0854485d64a6d8c9922a47ce631bbef05178 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Fri, 23 Oct 2026 17:38:12 -0400 Subject: [PATCH 05/11] test: add issue reproducers and return field nullability test for date_part --- .../functions/src/datetime/date_part.rs | 34 +++++++++++++++ .../spark/src/function/datetime/date_part.rs | 43 +++++++++++-------- .../test_files/datetime/date_part.slt | 18 +++++++- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 6871fdb7927c0..329b81ecc4daf 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -575,6 +575,8 @@ where let secs = as_int32_array(secs.as_ref())?; let fraction = date_part(array, part)?; let fraction = as_int32_array(fraction.as_ref())?; + // At most 60 seconds, which fits in Int32 for milliseconds and + // microseconds, so the narrowing `as_` never truncates let r: PrimitiveArray = binary(secs, fraction, |secs, fraction| { (secs as i64 * PER_SECOND + fraction as i64).as_() })?; @@ -864,6 +866,38 @@ mod tests { } } + /// `epoch` is NULL only for a NULL input, but the integer parts can be NULL + /// for a non-null input, so they are always nullable. + #[test] + fn return_field_nullability() { + for input_nullable in [false, true] { + let arg_fields: Vec = vec![ + Field::new("a", DataType::Utf8, false).into(), + Field::new( + "b", + DataType::Timestamp(TimeUnit::Second, None), + input_nullable, + ) + .into(), + ]; + for (part, nullable) in [ + ("epoch", input_nullable), + ("second", true), + ("millisecond", true), + ("nanosecond", true), + ] { + let part = ScalarValue::from(part); + let field = DatePartFunc::new() + .return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &[Some(&part), None], + }) + .unwrap(); + assert_eq!(field.is_nullable(), nullable, "{part}, {input_nullable}"); + } + } + } + #[test] fn timezone_parts_return_int32() { let arg_fields: Vec = vec![ diff --git a/datafusion/spark/src/function/datetime/date_part.rs b/datafusion/spark/src/function/datetime/date_part.rs index 8946ee90aad4e..67c63d52d1d6e 100644 --- a/datafusion/spark/src/function/datetime/date_part.rs +++ b/datafusion/spark/src/function/datetime/date_part.rs @@ -132,13 +132,14 @@ impl ScalarUDFImpl for SparkDatePart { mod tests { use super::*; use arrow::datatypes::TimeUnit; + use datafusion_functions::datetime::date_part; - /// The result field matches the field of the `date_part` it simplifies to. - /// That `date_part` can return NULL for a non-null input, so both are - /// nullable. #[test] - fn test_date_part_nullability_matches_simplified() -> Result<()> { - let arg_fields: Vec = vec![ + fn test_date_part_return_field_nullability_matches_date_part() { + let func = SparkDatePart::new(); + let part = ScalarValue::from("second"); + + let non_nullable_args: Vec = vec![ Arc::new(Field::new("part", DataType::Utf8, false)), Arc::new(Field::new( "ts", @@ -146,20 +147,24 @@ mod tests { false, )), ]; - let part = ScalarValue::from("second"); - let scalar_arguments = [Some(&part), None]; - let args = || ReturnFieldArgs { - arg_fields: &arg_fields, - scalar_arguments: &scalar_arguments, - }; - - let spark = SparkDatePart::new().return_field_from_args(args())?; - let simplified = - datafusion_functions::datetime::date_part().return_field_from_args(args())?; - assert!(spark.is_nullable()); - assert_eq!(spark.is_nullable(), simplified.is_nullable()); - assert_eq!(spark.data_type(), simplified.data_type()); - Ok(()) + let out = func + .return_field_from_args(ReturnFieldArgs { + arg_fields: &non_nullable_args, + scalar_arguments: &[Some(&part), None], + }) + .expect("non-nullable arg should succeed"); + assert_eq!(out.data_type(), &DataType::Int32); + assert!(out.is_nullable()); + + // Matches the `date_part` this simplifies to + let simplified = date_part() + .return_field_from_args(ReturnFieldArgs { + arg_fields: &non_nullable_args, + scalar_arguments: &[Some(&part), None], + }) + .expect("non-nullable arg should succeed"); + assert_eq!(out.data_type(), simplified.data_type()); + assert_eq!(out.is_nullable(), simplified.is_nullable()); } } diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index f0137d48d3cdd..4eacf7bf906b6 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -1262,6 +1262,21 @@ SELECT extract(millisecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')) ---- 2500 2500000 +# Reproducer from https://github.com/apache/datafusion/issues/26678 +query III +SELECT extract(nanosecond from arrow_cast(1500000000, 'Duration(Nanosecond)')), + extract(nanosecond from interval '1.5 seconds'), + extract(microsecond from interval '1.5 seconds') +---- +1500000000 1500000000 1500000 + +# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638 +query II +SELECT extract(millisecond from interval '62.5 seconds'), + extract(millisecond from arrow_cast(2500000000, 'Duration(Nanosecond)')) +---- +2500 2500 + # Reproducer from https://github.com/apache/datafusion/issues/14738 query III SELECT date_part('microseconds', interval '1 hour'), @@ -1500,7 +1515,8 @@ SELECT date_part('microsecond', timestamp '1970-01-01T00:40:00' - timestamp '197 ---- NULL -# Non-null inputs can give NULL results, so the result is nullable +# Non-null inputs can give NULL results, so the result is nullable. +# Reproducers from https://github.com/apache/datafusion/issues/26685 query I SELECT extract(second from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts) ---- From 8d5b5f1a2db45207e4d6146b735f31bb7addaeb3 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 24 Oct 2026 07:26:03 -0400 Subject: [PATCH 06/11] bench: add date_part sub-second cases for timestamps and times with nulls --- .../benches/datetime_expressions/date_part.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/datafusion/functions/benches/datetime_expressions/date_part.rs b/datafusion/functions/benches/datetime_expressions/date_part.rs index 48f9742dc763e..073a5fdd935da 100644 --- a/datafusion/functions/benches/datetime_expressions/date_part.rs +++ b/datafusion/functions/benches/datetime_expressions/date_part.rs @@ -188,6 +188,36 @@ fn generate_duration_nanosecond_array_with_nulls( ) } +fn generate_timestamp_ns_array_with_nulls(rng: &mut StdRng) -> TimestampNanosecondArray { + TimestampNanosecondArray::from( + (0..BATCH_SIZE) + .map(|_| { + if rng.random_bool(NULL_DENSITY) { + None + } else { + Some(rng.random_range(0..TS_BOUND * 1_000_000_000)) + } + }) + .collect::>(), + ) +} + +fn generate_time64_nanosecond_array_with_nulls( + rng: &mut StdRng, +) -> Time64NanosecondArray { + Time64NanosecondArray::from( + (0..BATCH_SIZE) + .map(|_| { + if rng.random_bool(NULL_DENSITY) { + None + } else { + Some(rng.random_range(0..SEC_DAY * 1_000_000_000)) + } + }) + .collect::>(), + ) +} + fn bench_date_part( c: &mut Criterion, udf: &Arc, @@ -247,6 +277,10 @@ fn criterion_benchmark(c: &mut Criterion) { let ts_ns_tz = Arc::new(generate_timestamp_ns_array(&mut rng).with_timezone("America/New_York")) as ArrayRef; + let ts_ns_nulls = + Arc::new(generate_timestamp_ns_array_with_nulls(&mut rng)) as ArrayRef; + let time64_ns_nulls = + Arc::new(generate_time64_nanosecond_array_with_nulls(&mut rng)) as ArrayRef; let udf = date_part(); @@ -378,8 +412,10 @@ fn criterion_benchmark(c: &mut Criterion) { ] { let mut arrays = vec![ ("ns_tz", &ts_ns_tz), + ("ns_nulls", &ts_ns_nulls), ("time32_ms", &time32_ms), ("time64_ns", &time64_ns), + ("time64_ns_nulls", &time64_ns_nulls), ("interval_dt", &interval_dt), ("interval_mdn", &interval_mdn), ("duration_ns_small", &duration_ns_small), From 2b4e8d7fd4fde0e24793972860bba2871960fb6d Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 24 Oct 2026 07:31:25 -0400 Subject: [PATCH 07/11] test: cover date_part sub-second parts per timestamp unit, times with nulls, and negative intervals --- .../test_files/datetime/date_part.slt | 121 ++++++++++++++++-- 1 file changed, 108 insertions(+), 13 deletions(-) diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index 4eacf7bf906b6..2deea198e889f 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -912,6 +912,43 @@ SELECT extract(epoch from arrow_cast('23:32:50.123456789'::time, 'Time64(Nanosec ---- 84770.123456789 +# sub-second parts of times, with a NULL row +query IIII +SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t) +FROM (SELECT arrow_cast(column1, 'Time32(Second)') AS t + FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time))) +---- +50 50000 50000000 50000000000 +NULL NULL NULL NULL +1 1000 1000000 1000000000 + +query IIII +SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t) +FROM (SELECT arrow_cast(column1, 'Time32(Millisecond)') AS t + FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time))) +---- +50 50123 50123000 50123000000 +NULL NULL NULL NULL +1 1000 1000000 1000000000 + +query IIII +SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t) +FROM (SELECT arrow_cast(column1, 'Time64(Microsecond)') AS t + FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time))) +---- +50 50123 50123456 50123456000 +NULL NULL NULL NULL +1 1000 1000000 1000000000 + +query IIII +SELECT extract(second from t), extract(millisecond from t), extract(microsecond from t), extract(nanosecond from t) +FROM (SELECT arrow_cast(column1, 'Time64(Nanosecond)') AS t + FROM (VALUES ('23:32:50.123456789'::time), (NULL), ('00:00:01.000000001'::time))) +---- +50 50123 50123456 50123456789 +NULL NULL NULL NULL +1 1000 1000000 1000000001 + # test_extract_epoch query R @@ -974,31 +1011,61 @@ SELECT extract(epoch from arrow_cast('1969-12-31', 'Date64')) ---- -86400 -# sub-second parts of timestamps before 1970 +# sub-second parts of timestamps in each unit, before and after 1970, with a NULL row query IIII SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) -FROM (VALUES - (timestamp '1969-12-31T23:59:58.123456789'), - (arrow_cast('1969-12-31T23:59:58.123', 'Timestamp(Millisecond, None)')), - (arrow_cast('1969-12-31T23:59:58', 'Timestamp(Second, None)')) -) AS t(ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Second, None)') AS ts + FROM (VALUES ('2020-09-08T12:00:12'), (NULL), ('1969-12-31T23:59:58'))) ---- -58 58123 58123456 58123456789 -58 58123 58123000 58123000000 +12 12000 12000000 12000000000 +NULL NULL NULL NULL 58 58000 58000000 58000000000 +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Millisecond, None)') AS ts + FROM (VALUES ('2020-09-08T12:00:12.123'), (NULL), ('1969-12-31T23:59:58.123'))) +---- +12 12123 12123000 12123000000 +NULL NULL NULL NULL +58 58123 58123000 58123000000 + +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Microsecond, None)') AS ts + FROM (VALUES ('2020-09-08T12:00:12.123456'), (NULL), ('1969-12-31T23:59:58.123456'))) +---- +12 12123 12123456 12123456000 +NULL NULL NULL NULL +58 58123 58123456 58123456000 + +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, None)') AS ts + FROM (VALUES ('2020-09-08T12:00:12.123456789'), (NULL), ('1969-12-31T23:59:58.123456789'))) +---- +12 12123 12123456 12123456789 +NULL NULL NULL NULL +58 58123 58123456 58123456789 + # Africa/Monrovia was UTC-00:44:30 until 1972, so the seconds depend on the timezone query IIII SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) -FROM (VALUES - (arrow_cast(500, 'Timestamp(Millisecond, Some("Africa/Monrovia"))')), - (arrow_cast(-1500, 'Timestamp(Millisecond, Some("Africa/Monrovia"))')), - (arrow_cast(500000000, 'Timestamp(Nanosecond, Some("Africa/Monrovia"))')) -) AS t(ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Millisecond, Some("Africa/Monrovia"))') AS ts + FROM (VALUES (500), (-1500), (NULL))) ---- 30 30500 30500000 30500000000 28 28500 28500000 28500000000 +NULL NULL NULL NULL + +query IIII +SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) +FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Africa/Monrovia"))') AS ts + FROM (VALUES (500000000), (-1500000000), (NULL))) +---- 30 30500 30500000 30500000000 +28 28500 28500000 28500000000 +NULL NULL NULL NULL # timestamps outside the range arrow represents are NULL, like the other parts query IIII @@ -1241,6 +1308,34 @@ SELECT extract(nanosecond from arrow_cast(NULL, 'Interval(DayTime)')) ---- NULL +# sub-second parts are the seconds field within the minute, with the sign of the +# milliseconds or nanoseconds field, whatever the days and months +query III +SELECT extract(millisecond from i), extract(microsecond from i), extract(nanosecond from i) +FROM (SELECT arrow_cast(column1, 'Interval(DayTime)') AS i FROM (VALUES + ('-62.5 seconds'), ('-60 seconds'), ('1 day 2 hours 5.1 seconds'), + ('-1 day 5.1 seconds'), ('-2 hours -5.1 seconds'), (NULL))) +---- +-2500 -2500000 -2500000000 +0 0 0 +5100 5100000 5100000000 +5100 5100000 5100000000 +-5100 -5100000 -5100000000 +NULL NULL NULL + +query III +SELECT extract(millisecond from i), extract(microsecond from i), extract(nanosecond from i) +FROM (SELECT arrow_cast(column1, 'Interval(MonthDayNano)') AS i FROM (VALUES + ('-62.5 seconds'), ('-60 seconds'), ('-59.999999999 seconds'), + ('1 month -62.5 seconds'), ('-1 day -62.5 seconds'), (NULL))) +---- +-2500 -2500000 -2500000000 +0 0 0 +-59999 -59999999 -59999999999 +-2500 -2500000 -2500000000 +-2500 -2500000 -2500000000 +NULL NULL NULL + # millisecond and microsecond keep the fraction past i32::MAX nanoseconds query IIII SELECT extract(millisecond from interval '62.5 seconds'), From 8736a1fe77472500f0b39476a17e7619a0bae61e Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 24 Oct 2026 07:36:02 -0400 Subject: [PATCH 08/11] refactor: simplify date_part Duration and Interval sub-second arms, test nullability with DESCRIBE --- .../functions/src/datetime/date_part.rs | 51 ++-------------- .../spark/src/function/datetime/date_part.rs | 59 +++--------------- .../test_files/datetime/date_part.slt | 60 ++++++++++--------- .../test_files/spark/datetime/date_part.slt | 7 +++ 4 files changed, 53 insertions(+), 124 deletions(-) diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 329b81ecc4daf..764aeb03115c2 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -531,21 +531,14 @@ fn subsecond_part(array: &dyn Array, part: DatePart) -> Result { as_interval_mdn_array(array)? .unary::<_, Int64Type>(|x| x.nanoseconds % 60_000_000_000), ), - (Duration(Nanosecond), DatePart::Nanosecond) => { - Arc::new(as_duration_nanosecond_array(array)?.reinterpret_cast::()) + // arrow's cast returns NULL on overflow, like its `date_part`, and + // reinterprets `Duration(Nanosecond)` as Int64 without copying + (Duration(_), DatePart::Nanosecond) => { + cast(&cast(array, &Duration(Nanosecond))?, &DataType::Int64)? } - // arrow's cast returns NULL on overflow, like its `date_part` - (Duration(_), DatePart::Nanosecond) => Arc::new( - as_duration_nanosecond_array(&cast(array, &Duration(Nanosecond))?)? - .reinterpret_cast::(), - ), // arrow's sub-second parts of intervals and durations already include - // the whole seconds - ( - Interval(ArrowIntervalUnit::DayTime | ArrowIntervalUnit::MonthDayNano) - | Duration(_), - _, - ) => date_part(array, part)?, + // the whole seconds, and it rejects them for `Interval(YearMonth)` + (Interval(_) | Duration(_), _) => date_part(array, part)?, (_, DatePart::Millisecond) => { seconds_with_fraction::(array, part)? } @@ -866,38 +859,6 @@ mod tests { } } - /// `epoch` is NULL only for a NULL input, but the integer parts can be NULL - /// for a non-null input, so they are always nullable. - #[test] - fn return_field_nullability() { - for input_nullable in [false, true] { - let arg_fields: Vec = vec![ - Field::new("a", DataType::Utf8, false).into(), - Field::new( - "b", - DataType::Timestamp(TimeUnit::Second, None), - input_nullable, - ) - .into(), - ]; - for (part, nullable) in [ - ("epoch", input_nullable), - ("second", true), - ("millisecond", true), - ("nanosecond", true), - ] { - let part = ScalarValue::from(part); - let field = DatePartFunc::new() - .return_field_from_args(ReturnFieldArgs { - arg_fields: &arg_fields, - scalar_arguments: &[Some(&part), None], - }) - .unwrap(); - assert_eq!(field.is_nullable(), nullable, "{part}, {input_nullable}"); - } - } - } - #[test] fn timezone_parts_return_int32() { let arg_fields: Vec = vec![ diff --git a/datafusion/spark/src/function/datetime/date_part.rs b/datafusion/spark/src/function/datetime/date_part.rs index 67c63d52d1d6e..ed09ab08fbc8c 100644 --- a/datafusion/spark/src/function/datetime/date_part.rs +++ b/datafusion/spark/src/function/datetime/date_part.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::DataType; use datafusion_common::types::logical_date; use datafusion_common::{ Result, ScalarValue, internal_err, types::logical_string, utils::take_function_args, @@ -23,10 +23,9 @@ use datafusion_common::{ use datafusion_expr::expr::ScalarFunction; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::{ - Coercion, ColumnarValue, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, - Signature, TypeSignature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignature, TypeSignatureClass, Volatility, }; -use std::sync::Arc; /// Wrapper around datafusion date_part function to handle /// Spark behavior returning day of the week 1-indexed instead of 0-indexed and different part aliases. @@ -77,14 +76,11 @@ impl ScalarUDFImpl for SparkDatePart { &self.signature } + // The default `return_field_from_args` makes the result nullable, like the + // integer parts of `date_part`, which this simplifies to, because they can + // be NULL for a non-null input fn return_type(&self, _arg_types: &[DataType]) -> Result { - internal_err!("Use return_field_from_args in this case instead.") - } - - fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { - // Nullable like the integer parts of `date_part`, which this simplifies - // to, because they can be NULL for a non-null input - Ok(Arc::new(Field::new(self.name(), DataType::Int32, true))) + Ok(DataType::Int32) } fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { @@ -127,44 +123,3 @@ impl ScalarUDFImpl for SparkDatePart { Ok(ExprSimplifyResult::Simplified(date_part_expr)) } } - -#[cfg(test)] -mod tests { - use super::*; - use arrow::datatypes::TimeUnit; - use datafusion_functions::datetime::date_part; - - #[test] - fn test_date_part_return_field_nullability_matches_date_part() { - let func = SparkDatePart::new(); - let part = ScalarValue::from("second"); - - let non_nullable_args: Vec = vec![ - Arc::new(Field::new("part", DataType::Utf8, false)), - Arc::new(Field::new( - "ts", - DataType::Timestamp(TimeUnit::Second, None), - false, - )), - ]; - - let out = func - .return_field_from_args(ReturnFieldArgs { - arg_fields: &non_nullable_args, - scalar_arguments: &[Some(&part), None], - }) - .expect("non-nullable arg should succeed"); - assert_eq!(out.data_type(), &DataType::Int32); - assert!(out.is_nullable()); - - // Matches the `date_part` this simplifies to - let simplified = date_part() - .return_field_from_args(ReturnFieldArgs { - arg_fields: &non_nullable_args, - scalar_arguments: &[Some(&part), None], - }) - .expect("non-nullable arg should succeed"); - assert_eq!(out.data_type(), simplified.data_type()); - assert_eq!(out.is_nullable(), simplified.is_nullable()); - } -} diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index 2deea198e889f..e29115b4bbf96 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -1264,7 +1264,8 @@ statement ok drop table t; # nanosecond on an interval is the seconds field (within the minute) in nanoseconds, -# like microsecond and millisecond +# like microsecond and millisecond. +# Reproducer from https://github.com/apache/datafusion/issues/26678 query III SELECT extract(nanosecond from interval '1.5 seconds'), extract(microsecond from interval '1.5 seconds'), @@ -1336,7 +1337,8 @@ FROM (SELECT arrow_cast(column1, 'Interval(MonthDayNano)') AS i FROM (VALUES -2500 -2500000 -2500000000 NULL NULL NULL -# millisecond and microsecond keep the fraction past i32::MAX nanoseconds +# millisecond and microsecond keep the fraction past i32::MAX nanoseconds. +# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638 query IIII SELECT extract(millisecond from interval '62.5 seconds'), extract(microsecond from interval '62.5 seconds'), @@ -1357,21 +1359,6 @@ SELECT extract(millisecond from arrow_cast('62.5 seconds', 'Interval(DayTime)')) ---- 2500 2500000 -# Reproducer from https://github.com/apache/datafusion/issues/26678 -query III -SELECT extract(nanosecond from arrow_cast(1500000000, 'Duration(Nanosecond)')), - extract(nanosecond from interval '1.5 seconds'), - extract(microsecond from interval '1.5 seconds') ----- -1500000000 1500000000 1500000 - -# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638 -query II -SELECT extract(millisecond from interval '62.5 seconds'), - extract(millisecond from arrow_cast(2500000000, 'Duration(Nanosecond)')) ----- -2500 2500 - # Reproducer from https://github.com/apache/datafusion/issues/14738 query III SELECT date_part('microseconds', interval '1 hour'), @@ -1380,12 +1367,9 @@ SELECT date_part('microseconds', interval '1 hour'), ---- 0 0 0 -query error DataFusion error: Arrow error: Compute error: Second does not support: Interval\(YearMonth\) +query error DataFusion error: Arrow error: Compute error: Nanosecond does not support: Interval\(YearMonth\) SELECT extract(nanosecond from arrow_cast('10 months', 'Interval(YearMonth)')) -query error DataFusion error: Arrow error: Compute error: Second does not support: Interval\(YearMonth\) -SELECT extract(millisecond from arrow_cast('10 months', 'Interval(YearMonth)')) - # test_extract_duration query I @@ -1514,7 +1498,8 @@ SELECT extract(day from arrow_cast(NULL, 'Duration(Second)')) ---- NULL -# nanosecond on a duration is its total length in nanoseconds +# nanosecond on a duration is its total length in nanoseconds. +# Reproducer from https://github.com/apache/datafusion/issues/26678 query IIII SELECT extract(nanosecond from arrow_cast(1500000000, 'Duration(Nanosecond)')), extract(nanosecond from arrow_cast(1500000, 'Duration(Microsecond)')), @@ -1563,12 +1548,13 @@ FROM (VALUES (9223372036, -9223372036854), (9223372037, -9223372036855), (NULL, NULL NULL NULL NULL -# millisecond and microsecond on a duration are its total length in that unit +# millisecond and microsecond on a duration are its total length in that unit. +# Reproducer from https://github.com/apache/datafusion/issues/14738#issuecomment-6372885638 query II -SELECT extract(millisecond from arrow_cast(5500000000, 'Duration(Nanosecond)')), - extract(microsecond from arrow_cast(5500000000, 'Duration(Nanosecond)')) +SELECT extract(millisecond from arrow_cast(2500000000, 'Duration(Nanosecond)')), + extract(microsecond from arrow_cast(2500000000, 'Duration(Nanosecond)')) ---- -5500 5500000 +2500 2500000 # truncates toward zero, like the other parts of a duration query III @@ -1604,7 +1590,9 @@ NULL NULL NULL NULL NULL NULL -# Reproducer from https://github.com/apache/datafusion/issues/14738: 2.4e9 microseconds don't fit in Int32 +# Reproducer from https://github.com/apache/datafusion/issues/14738. Postgres returns 0, because +# there the difference is an interval. Here it is a Duration, whose microsecond is its total +# length, and 2.4e9 microseconds don't fit in Int32. query I SELECT date_part('microsecond', timestamp '1970-01-01T00:40:00' - timestamp '1970-01-01T00:00:00') ---- @@ -1632,6 +1620,24 @@ SELECT extract(millisecond from d) FROM (VALUES (arrow_cast(2147483648, 'Duratio ---- NULL +# epoch is NULL only for a NULL input, so it keeps the input's nullability +query TTT +DESCRIBE SELECT extract(epoch from ts) AS e, extract(second from ts) AS s, + extract(millisecond from ts) AS ms, extract(nanosecond from ts) AS ns +FROM (VALUES (timestamp '2020-01-01')) AS t(ts) +---- +e Float64 NO +s Int32 YES +ms Int32 YES +ns Int64 YES + +query TTT +DESCRIBE SELECT extract(epoch from ts) AS e, extract(second from ts) AS s +FROM (VALUES (timestamp '2020-01-01'), (NULL)) AS t(ts) +---- +e Float64 YES +s Int32 YES + # test_extract_date_part_func query B diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt index beece06d66bbf..566179ba14f0c 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt @@ -284,3 +284,10 @@ FROM VALUES 1999 2000 NULL + +# Nullable even for a non-null input, like the integer parts of the date_part +# this simplifies to +query TTT +DESCRIBE SELECT date_part('SECOND'::string, ts) AS s FROM (VALUES (timestamp '2020-01-01')) AS t(ts) +---- +s Int32 YES From 9745f21b9dff37e0e9dfc5872e85abc42b6709a2 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 24 Oct 2026 08:35:46 -0400 Subject: [PATCH 09/11] perf: compute date_part sub-second parts of timestamps and times in one pass --- .../functions/src/datetime/date_part.rs | 170 +++++++++++++++--- .../test_files/datetime/date_part.slt | 15 ++ 2 files changed, 158 insertions(+), 27 deletions(-) diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 764aeb03115c2..6cb5d5cf60eda 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -18,22 +18,27 @@ use std::str::FromStr; use std::sync::Arc; -use arrow::array::temporal_conversions::as_datetime_with_timezone; +use arrow::array::temporal_conversions::{ + as_datetime_with_timezone, timestamp_ms_to_datetime, timestamp_ns_to_datetime, + timestamp_s_to_datetime, timestamp_us_to_datetime, +}; use arrow::array::timezone::Tz; use arrow::array::{ Array, ArrayRef, Float64Array, Int32Array, Int64Array, PrimitiveArray, }; -use arrow::compute::{DatePart, binary, cast, date_part}; +use arrow::compute::{DatePart, cast, date_part}; use arrow::datatypes::DataType::{ Date32, Date64, Duration, Interval, Time32, Time64, Timestamp, }; use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; use arrow::datatypes::{ ArrowPrimitiveType, ArrowTimestampType, DataType, Field, FieldRef, Int32Type, - Int64Type, IntervalUnit as ArrowIntervalUnit, TimestampMicrosecondType, - TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, + Int64Type, IntervalUnit as ArrowIntervalUnit, Time32MillisecondType, + Time32SecondType, Time64MicrosecondType, Time64NanosecondType, + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, }; -use chrono::{Datelike, NaiveDate, Offset}; +use chrono::{Datelike, NaiveDate, NaiveDateTime, Offset, TimeZone, Timelike, Utc}; use datafusion_common::types::{NativeType, logical_date}; use num_traits::AsPrimitive; @@ -43,12 +48,11 @@ use datafusion_common::{ cast::{ as_date32_array, as_date64_array, as_duration_microsecond_array, as_duration_millisecond_array, as_duration_nanosecond_array, - as_duration_second_array, as_int32_array, as_interval_dt_array, - as_interval_mdn_array, as_interval_ym_array, as_primitive_array, - as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array, - as_time64_nanosecond_array, as_timestamp_microsecond_array, - as_timestamp_millisecond_array, as_timestamp_nanosecond_array, - as_timestamp_second_array, + as_duration_second_array, as_interval_dt_array, as_interval_mdn_array, + as_interval_ym_array, as_primitive_array, as_time32_millisecond_array, + as_time32_second_array, as_time64_microsecond_array, as_time64_nanosecond_array, + as_timestamp_microsecond_array, as_timestamp_millisecond_array, + as_timestamp_nanosecond_array, as_timestamp_second_array, }, exec_err, internal_err, types::logical_string, @@ -509,9 +513,10 @@ fn epoch(array: &dyn Array) -> Result { /// minute for timestamps, times and intervals, and the total length for /// durations. /// -/// `nanosecond` returns an Int64 rather than Int32 because there are 1 billion -/// `nanosecond`s in each second, so representing up to 60 seconds as -/// nanoseconds can be values up to 60 billion, which does not fit in Int32. +/// `nanosecond` returns Int64, because 60 seconds are 60 billion nanoseconds +/// and a duration can be up to `i64::MAX` nanoseconds long. `millisecond` and +/// `microsecond` return Int32. A duration whose length overflows the return +/// type is NULL. fn subsecond_part(array: &dyn Array, part: DatePart) -> Result { let len = array.len(); let nulls = array.nulls().cloned(); @@ -552,9 +557,8 @@ fn subsecond_part(array: &dyn Array, part: DatePart) -> Result { }) } -/// Combine arrow's seconds of the minute with its fraction of the second in -/// `part` units, for types where arrow returns them separately (timestamps -/// and times). +/// Return the seconds of the minute of timestamps (in their timezone) and +/// times, with their fraction, in units of `1 / PER_SECOND` seconds. fn seconds_with_fraction( array: &dyn Array, part: DatePart, @@ -563,16 +567,128 @@ where O: ArrowPrimitiveType, i64: AsPrimitive, { - let secs = date_part(array, DatePart::Second)?; - // This assumes array is primitive and not a dictionary - let secs = as_int32_array(secs.as_ref())?; - let fraction = date_part(array, part)?; - let fraction = as_int32_array(fraction.as_ref())?; - // At most 60 seconds, which fits in Int32 for milliseconds and - // microseconds, so the narrowing `as_` never truncates - let r: PrimitiveArray = binary(secs, fraction, |secs, fraction| { - (secs as i64 * PER_SECOND + fraction as i64).as_() - })?; + Ok(match array.data_type() { + Timestamp(unit, tz) => { + let tz = parse_tz(tz.as_ref())?; + match unit { + Second => timestamp_seconds_with_fraction::< + TimestampSecondType, + O, + PER_SECOND, + >(array, tz, timestamp_s_to_datetime)?, + Millisecond => timestamp_seconds_with_fraction::< + TimestampMillisecondType, + O, + PER_SECOND, + >(array, tz, timestamp_ms_to_datetime)?, + Microsecond => timestamp_seconds_with_fraction::< + TimestampMicrosecondType, + O, + PER_SECOND, + >(array, tz, timestamp_us_to_datetime)?, + Nanosecond => timestamp_seconds_with_fraction::< + TimestampNanosecondType, + O, + PER_SECOND, + >(array, tz, timestamp_ns_to_datetime)?, + } + } + Time32(Second) => { + time_seconds_with_fraction::(array)? + } + Time32(Millisecond) => { + time_seconds_with_fraction::( + array, + )? + } + Time64(Microsecond) => { + time_seconds_with_fraction::( + array, + )? + } + Time64(Nanosecond) => time_seconds_with_fraction::< + Time64NanosecondType, + O, + 1_000_000_000, + PER_SECOND, + >(array)?, + other => return exec_err!("{part} does not support {other}"), + }) +} + +fn timestamp_seconds_with_fraction( + array: &dyn Array, + tz: Option, + // arrow's converter for `T`, as its `date_part` uses. The generic + // `as_datetime::` measured about 1.5x slower. + to_datetime: impl Fn(i64) -> Option, +) -> Result +where + T: ArrowTimestampType, + O: ArrowPrimitiveType, + i64: AsPrimitive, +{ + // Like arrow's `date_part`, but one chrono conversion per row instead of + // one each for the seconds and the fraction + fn seconds( + t: &impl Timelike, + ) -> O::Native + where + i64: AsPrimitive, + { + // At most 60 seconds, which fits in Int32 for milliseconds and + // microseconds, so the narrowing `as_` never truncates + (t.second() as i64 * PER_SECOND + + t.nanosecond() as i64 / (1_000_000_000 / PER_SECOND)) + .as_() + } + + let array = as_primitive_array::(array)?; + // `unary_opt` keeps the input nulls and additionally nulls out any value + // that cannot be represented as a `DateTime` (out of chrono's range). + let r: PrimitiveArray = match tz { + Some(tz) => array.unary_opt(|v| { + to_datetime(v).map(|naive| { + seconds::( + &Utc.from_utc_datetime(&naive).with_timezone(&tz), + ) + }) + }), + None => array + .unary_opt(|v| to_datetime(v).map(|naive| seconds::(&naive))), + }; + Ok(Arc::new(r)) +} + +/// Like [`timestamp_seconds_with_fraction`] for times, which count +/// `1 / UNITS_PER_SECOND` seconds since midnight. +fn time_seconds_with_fraction( + array: &dyn Array, +) -> Result +where + T: ArrowPrimitiveType, + T::Native: AsPrimitive, + O: ArrowPrimitiveType, + i64: AsPrimitive, +{ + const SECONDS_PER_DAY: i64 = 86_400; + let array = as_primitive_array::(array)?; + // Values outside one day are NULL, as in arrow's `date_part`. arrow's + // `as_time` would wrap some of them around to a valid time instead. + let r: PrimitiveArray = array.unary_opt(|v| { + let v: i64 = v.as_(); + (0..SECONDS_PER_DAY * UNITS_PER_SECOND) + .contains(&v) + .then(|| { + let v = v % (60 * UNITS_PER_SECOND); + if PER_SECOND >= UNITS_PER_SECOND { + v * (PER_SECOND / UNITS_PER_SECOND) + } else { + v / (UNITS_PER_SECOND / PER_SECOND) + } + .as_() + }) + }); Ok(Arc::new(r)) } diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index e29115b4bbf96..7fcfa77825241 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -949,6 +949,21 @@ FROM (SELECT arrow_cast(column1, 'Time64(Nanosecond)') AS t NULL NULL NULL NULL 1 1000 1000000 1000000001 +# times outside one day are NULL, like the other parts. The last row is 2^32 + 5 +# seconds, which must not wrap around to 5 seconds. +query IIII +SELECT extract(nanosecond from arrow_cast(arrow_cast(column1, 'Int32'), 'Time32(Second)')), + extract(nanosecond from arrow_cast(arrow_cast(column2, 'Int32'), 'Time32(Millisecond)')), + extract(nanosecond from arrow_cast(column3, 'Time64(Microsecond)')), + extract(nanosecond from arrow_cast(column4, 'Time64(Nanosecond)')) +FROM (VALUES (-1, -1, -1, -1), + (86400, 86400000, 86400000000, 86400000000000), + (NULL, NULL, 4294967301000000, 4294967301000000000)) +---- +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL + # test_extract_epoch query R From 48ff4d8da0d3a990f6ef216e13ef4e1f5cbe758f Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 24 Oct 2026 08:37:28 -0400 Subject: [PATCH 10/11] refactor: reuse arrow's time constants in date_part sub-second helpers --- .../functions/src/datetime/date_part.rs | 82 ++++++++++--------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 6cb5d5cf60eda..1872021abcb44 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -19,8 +19,9 @@ use std::str::FromStr; use std::sync::Arc; use arrow::array::temporal_conversions::{ - as_datetime_with_timezone, timestamp_ms_to_datetime, timestamp_ns_to_datetime, - timestamp_s_to_datetime, timestamp_us_to_datetime, + MICROSECONDS, MILLISECONDS, NANOSECONDS, SECONDS_IN_DAY, as_datetime_with_timezone, + timestamp_ms_to_datetime, timestamp_ns_to_datetime, timestamp_s_to_datetime, + timestamp_us_to_datetime, }; use arrow::array::timezone::Tz; use arrow::array::{ @@ -545,13 +546,13 @@ fn subsecond_part(array: &dyn Array, part: DatePart) -> Result { // the whole seconds, and it rejects them for `Interval(YearMonth)` (Interval(_) | Duration(_), _) => date_part(array, part)?, (_, DatePart::Millisecond) => { - seconds_with_fraction::(array, part)? + seconds_with_fraction::(array, part)? } (_, DatePart::Microsecond) => { - seconds_with_fraction::(array, part)? + seconds_with_fraction::(array, part)? } (_, DatePart::Nanosecond) => { - seconds_with_fraction::(array, part)? + seconds_with_fraction::(array, part)? } (_, part) => return internal_err!("{part} is not a sub-second part"), }) @@ -596,23 +597,24 @@ where Time32(Second) => { time_seconds_with_fraction::(array)? } - Time32(Millisecond) => { - time_seconds_with_fraction::( - array, - )? - } - Time64(Microsecond) => { - time_seconds_with_fraction::( - array, - )? - } - Time64(Nanosecond) => time_seconds_with_fraction::< - Time64NanosecondType, + Time32(Millisecond) => time_seconds_with_fraction::< + Time32MillisecondType, + O, + MILLISECONDS, + PER_SECOND, + >(array)?, + Time64(Microsecond) => time_seconds_with_fraction::< + Time64MicrosecondType, O, - 1_000_000_000, + MICROSECONDS, PER_SECOND, >(array)?, - other => return exec_err!("{part} does not support {other}"), + Time64(Nanosecond) => { + time_seconds_with_fraction::( + array, + )? + } + other => return internal_err!("{part} does not support {other}"), }) } @@ -628,38 +630,39 @@ where O: ArrowPrimitiveType, i64: AsPrimitive, { - // Like arrow's `date_part`, but one chrono conversion per row instead of - // one each for the seconds and the fraction - fn seconds( - t: &impl Timelike, - ) -> O::Native - where - i64: AsPrimitive, - { - // At most 60 seconds, which fits in Int32 for milliseconds and - // microseconds, so the narrowing `as_` never truncates - (t.second() as i64 * PER_SECOND - + t.nanosecond() as i64 / (1_000_000_000 / PER_SECOND)) - .as_() - } - let array = as_primitive_array::(array)?; - // `unary_opt` keeps the input nulls and additionally nulls out any value + // One chrono conversion per row, where arrow's `date_part` needs one each + // for the seconds and the fraction. `unary_opt` keeps the input nulls and additionally nulls out any value // that cannot be represented as a `DateTime` (out of chrono's range). let r: PrimitiveArray = match tz { Some(tz) => array.unary_opt(|v| { to_datetime(v).map(|naive| { - seconds::( + seconds_of_minute::( &Utc.from_utc_datetime(&naive).with_timezone(&tz), ) }) }), - None => array - .unary_opt(|v| to_datetime(v).map(|naive| seconds::(&naive))), + None => array.unary_opt(|v| { + to_datetime(v).map(|naive| seconds_of_minute::(&naive)) + }), }; Ok(Arc::new(r)) } +/// The seconds of the minute of `t`, with their fraction, in units of +/// `1 / PER_SECOND` seconds +fn seconds_of_minute( + t: &impl Timelike, +) -> O::Native +where + i64: AsPrimitive, +{ + // At most 60 seconds, which fits in Int32 for milliseconds and + // microseconds, so the narrowing `as_` never truncates + (t.second() as i64 * PER_SECOND + t.nanosecond() as i64 / (NANOSECONDS / PER_SECOND)) + .as_() +} + /// Like [`timestamp_seconds_with_fraction`] for times, which count /// `1 / UNITS_PER_SECOND` seconds since midnight. fn time_seconds_with_fraction( @@ -671,13 +674,12 @@ where O: ArrowPrimitiveType, i64: AsPrimitive, { - const SECONDS_PER_DAY: i64 = 86_400; let array = as_primitive_array::(array)?; // Values outside one day are NULL, as in arrow's `date_part`. arrow's // `as_time` would wrap some of them around to a valid time instead. let r: PrimitiveArray = array.unary_opt(|v| { let v: i64 = v.as_(); - (0..SECONDS_PER_DAY * UNITS_PER_SECOND) + (0..SECONDS_IN_DAY * UNITS_PER_SECOND) .contains(&v) .then(|| { let v = v % (60 * UNITS_PER_SECOND); From 6b2083bb7f23df13e8f64d2afcdd38ecfebb5f92 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 24 Oct 2026 08:58:56 -0400 Subject: [PATCH 11/11] docs: link apache/arrow-rs#11893 in date_part time helper comment --- datafusion/functions/src/datetime/date_part.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 1872021abcb44..919f1b0ac8b53 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -675,8 +675,8 @@ where i64: AsPrimitive, { let array = as_primitive_array::(array)?; - // Values outside one day are NULL, as in arrow's `date_part`. arrow's - // `as_time` would wrap some of them around to a valid time instead. + // Values outside one day are NULL, as in arrow's `date_part`. `as_time` + // wraps some to a valid time: https://github.com/apache/arrow-rs/issues/11893 let r: PrimitiveArray = array.unary_opt(|v| { let v: i64 = v.as_(); (0..SECONDS_IN_DAY * UNITS_PER_SECOND)