diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs index 981cc0e5ced..aaeec729c90 100644 --- a/native/spark-expr/src/conversion_funcs/string.rs +++ b/native/spark-expr/src/conversion_funcs/string.rs @@ -1373,6 +1373,30 @@ pub(crate) fn ymd_to_epoch_day(year: i64, month: i64, day: i64) -> Option { Some(days_from_civil(year, month, day)) } +/// chrono 0.4.45 NaiveDate year bounds (`MIN_YEAR` / `MAX_YEAR`). +const CHRONO_MIN_YEAR: i64 = -262143; +const CHRONO_MAX_YEAR: i64 = 262142; + +/// Offset (local − UTC) in seconds at `naive` in `tz`. +/// +/// Fall-back overlap picks the earlier (pre-transition) offset, matching Spark. +/// Spring-forward gaps probe three hours earlier with a checked subtract so extreme +/// NaiveDateTimes cannot panic; a second `None` yields `None` (caller may treat as UTC). +fn offset_secs_at(tz: &T, naive: chrono::NaiveDateTime) -> Option { + match tz.from_local_datetime(&naive) { + LocalResult::Single(dt) => Some(dt.offset().fix().local_minus_utc() as i64), + LocalResult::Ambiguous(dt, _) => Some(dt.offset().fix().local_minus_utc() as i64), + LocalResult::None => { + let pre = naive.checked_sub_signed(chrono::Duration::hours(3))?; + match tz.from_local_datetime(&pre) { + LocalResult::Single(dt) => Some(dt.offset().fix().local_minus_utc() as i64), + LocalResult::Ambiguous(dt, _) => Some(dt.offset().fix().local_minus_utc() as i64), + LocalResult::None => None, + } + } + } +} + fn parse_timestamp_to_micros( timestamp_info: &TimeStampInfo, tz: &T, @@ -1411,14 +1435,15 @@ fn parse_timestamp_to_micros( // Java's ZonedDateTime.of() advances by the gap length, which is equivalent to // utc = local_naive − pre_gap_offset LocalResult::None => { - let probe = local_naive - chrono::Duration::hours(3); - let pre_offset = match tz.from_local_datetime(&probe) { - LocalResult::Single(dt) => dt.offset().fix(), - LocalResult::Ambiguous(dt, _) => dt.offset().fix(), - LocalResult::None => return Ok(None), + let offset_secs = match offset_secs_at(tz, local_naive) { + Some(s) => s, + None => return Ok(None), }; - let offset_secs = pre_offset.local_minus_utc() as i64; - let utc_naive = local_naive - chrono::Duration::seconds(offset_secs); + let utc_naive = + match local_naive.checked_sub_signed(chrono::Duration::seconds(offset_secs)) { + Some(n) => n, + None => return Ok(None), + }; Some(utc_naive.and_utc().timestamp_micros()) } }; @@ -1427,13 +1452,14 @@ fn parse_timestamp_to_micros( } else { // NaiveDate::from_ymd_opt returned None. This means either: // (a) invalid calendar date (Feb 29 on non-leap year, month 13, etc.) - // (b) year outside chrono's representable range (> 262143 or < -262144) + // (b) year outside chrono's representable range (chrono 0.4.45: outside + // [-262143, 262142]) // // For case (b) we fall back to Howard Hinnant's direct arithmetic, which works // for any year that fits in i64. This covers the Long.MaxValue / Long.MinValue // boundary timestamps (year 294247 / -290308). let year = timestamp_info.year as i64; - if (-262144..=262143).contains(&year) { + if (CHRONO_MIN_YEAR..=CHRONO_MAX_YEAR).contains(&year) { // Year is in chrono's range but date was rejected -> truly invalid date. return Ok(None); } @@ -1443,18 +1469,31 @@ fn parse_timestamp_to_micros( Some(days) => days, None => return Ok(None), }; - // Compute the timezone offset using epoch as a surrogate probe point. - // Extreme-year timestamps are only valid with a UTC-like fixed offset (any DST - // zone would overflow). Using epoch gives us the standard offset. - let epoch_probe = NaiveDate::from_ymd_opt(1970, 1, 1) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap(); - let tz_offset_secs: i64 = match tz.from_local_datetime(&epoch_probe) { - LocalResult::Single(dt) => dt.offset().fix().local_minus_utc() as i64, - LocalResult::Ambiguous(dt, _) => dt.offset().fix().local_minus_utc() as i64, - LocalResult::None => 0, + // Look up the zone offset at the same local civil time on a year inside chrono's + // range that is congruent modulo the 400-year Gregorian cycle (leap days match). + // Far-past and far-future need different eras: a single 1600..=1999 window can + // land after a zone's LMT→standard transition (e.g. LA 1883, Tokyo ~1888) for + // some far-past residues, and still in the LMT era for far-future residues + // (262543 → 1743), disagreeing with Spark/Java which use the first offset before + // any transition and the modern/standard offset for post-horizon dates. + // far-past → 400..=799 (before every IANA transition we care about → LMT) + // far-future → 2000..=2399 (modern standard; January matches Spark; summer DST + // after chrono-tz's last timespan remains a separate gap #6816) + // Fixed offsets and UTC are unchanged either way. + let rem = year.rem_euclid(400); + let probe_year = if year > CHRONO_MAX_YEAR { + (2000 + rem) as i32 + } else { + (400 + rem) as i32 }; + let probe_date = + NaiveDate::from_ymd_opt(probe_year, timestamp_info.month, timestamp_info.day) + .expect("month/day valid across a 400-year shift"); + let probe_naive = probe_date.and_time(naive_time); + // None/Ambiguous are rare for this probe (it lands before every transition or + // after chrono-tz's last timespan for most zones), but the helper covers DST gaps + // if the congruent civil time falls in one. A second None falls back to UTC (0). + let tz_offset_secs = offset_secs_at(tz, probe_naive).unwrap_or(0); // Compute seconds since epoch via direct calendar arithmetic. // Use i128 for the intermediate multiply-by-1_000_000 step: the seconds value can be // just outside the i64 range while the final microseconds result is still within range @@ -3091,6 +3130,89 @@ mod tests { ); } + #[test] + fn extreme_year_session_tz_uses_historic_offset() { + // Session-TZ (no zone suffix) extremes: far-past uses historic / LMT offset; + // far-future uses modern/standard offset (matching Spark/Java ZoneRules). + // Spark's America/Los_Angeles uses LMT -07:52:58 before 1883 and -08:00 in + // January after; Asia/Tokyo uses LMT +09:18:59 then +09:00. A single + // 1600+rem probe mixed those eras (issue #7210 / review on #7255). + // Spark-backed coverage lives in CometDateTimeUtilsSuite. + let la = &Tz::from_str("America/Los_Angeles").unwrap(); + let tokyo = &Tz::from_str("Asia/Tokyo").unwrap(); + // Far-past Long.MinValue civil: LMT -07:52:58 → +28378s vs UTC parse. + assert_eq!( + timestamp_parser("-290308-12-21 19:59:05.224192", EvalMode::Legacy, la, true).unwrap(), + Some(-9223372008476775808), + ); + // Far-past residue that lands after 1883 under a 1600+rem probe (1900) must + // still get LMT, not PST/JST. + assert_eq!( + timestamp_parser("-262500-01-01", EvalMode::Legacy, la, true).unwrap(), + Some(-8345867069222000000), + ); + assert_eq!( + timestamp_parser("-262500-01-01", EvalMode::Legacy, tokyo, true).unwrap(), + Some(-8345867131139000000), + ); + assert_eq!( + timestamp_parser("-262544-01-01", EvalMode::Legacy, la, true).unwrap(), + Some(-8347255603622000000), + ); + // Far-future January: modern standard (-08:00 / +09:00), not LMT. + assert_eq!( + timestamp_parser("262543-01-01", EvalMode::Legacy, la, true).unwrap(), + Some(8222889686400000000), + ); + assert_eq!( + timestamp_parser("262543-01-01", EvalMode::Legacy, tokyo, true).unwrap(), + Some(8222889625200000000), + ); + let plus2 = &Tz::from_str("+02:00").unwrap(); + // Local = UTC+2, so the UTC instant is 2h earlier than the UTC-session parse. + assert_eq!( + timestamp_parser( + "294247-01-10 04:00:54.775807", + EvalMode::Legacy, + plus2, + true + ) + .unwrap(), + Some(i64::MAX - 2 * 3600 * 1_000_000), + ); + } + + #[test] + + fn extreme_year_chrono_bound_neighbours_do_not_panic() { + // chrono 0.4.45 accepts [-262143, 262142]. Years just outside that used to + // land on 262143 / -262144 after the old while-loop shift and panic in + // from_ymd_opt.expect. rem_euclid keeps the probe inside the range. + let utc = &Tz::from_str("UTC").unwrap(); + for (s, year) in [ + ("262543-01-01", 262543i64), + ("262542-01-01", 262542), + ("262143-01-01", 262143), + ("262142-01-01", 262142), + ("-262544-01-01", -262544), + ("-262543-01-01", -262543), + ("-262144-01-01", -262144), + ("-262143-01-01", -262143), + ] { + let days = ymd_to_epoch_day(year, 1, 1).unwrap(); + let expected = days + .checked_mul(86400) + .unwrap() + .checked_mul(1_000_000) + .unwrap(); + assert_eq!( + timestamp_parser(s, EvalMode::Legacy, utc, true).unwrap(), + Some(expected), + "{s}" + ); + } + } + #[test] fn test_leading_whitespace_t_hm() { let tz = &Tz::from_str("UTC").unwrap(); diff --git a/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala b/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala index 072009d7ab6..44659f10bfe 100644 --- a/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala @@ -307,6 +307,28 @@ class CometDateTimeUtilsSuite extends CometTestBase { } } + test("extreme-year session TZ uses historic zone offset") { + // Session-TZ casts of years outside chrono's NaiveDate range: far-past uses the + // zone's historic / LMT offset; far-future uses modern/standard. Compare Comet + // to Spark via Long (seconds) so the 422s America/Los_Angeles LMT vs PST gap + // survives truncation and toJavaTimestamp range limits are avoided. Also covers + // years just outside chrono's [-262143, 262142] bounds that previously panicked. + for (tz <- Seq("America/Los_Angeles", "Asia/Tokyo")) { + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz) { + checkCastToTimestampAsLong( + Seq( + "-290308-12-21 19:59:05.224192", + "-262544-01-01 00:00:00", + "-262543-01-01 00:00:00", + // Residue that under a 1600+rem probe lands after LMT→standard (1900). + "-262500-01-01 00:00:00", + "262542-01-01 00:00:00", + // Far-future January must use modern standard offset, not LMT. + "262543-01-01 00:00:00")) + } + } + } + test("SPARK-37326: cast string to TIMESTAMP_NTZ rejects timezone offsets") { checkCastToTimestampNTZ( Seq(