Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions crates/iceberg/src/spec/partition.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1951,6 +1951,32 @@ mod tests {
);
}

#[test]
fn test_partition_to_path_float_matches_java() {
let schema = Schema::builder()
.with_fields(vec![
NestedField::required(1, "d", Type::Primitive(PrimitiveType::Double)).into(),
NestedField::required(2, "f", Type::Primitive(PrimitiveType::Float)).into(),
])
.build()
.unwrap();

let spec = PartitionSpec::builder(schema.clone())
.add_partition_field("d", "d", Transform::Identity)
.unwrap()
.add_partition_field("f", "f", Transform::Identity)
.unwrap()
.build()
.unwrap();

let data = Struct::from_iter([Some(Literal::double(1.0)), Some(Literal::float(1.0e7_f32))]);

assert_eq!(
spec.partition_to_path(&data, schema.into()),
"d=1.0/f=1.0E7"
);
}

#[test]
fn test_partition_to_path_escaped_strings() {
let schema = Schema::builder()
Expand Down
57 changes: 56 additions & 1 deletion crates/iceberg/src/spec/values/datum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
//! Typed literals with validation

use std::cmp::Ordering;
use std::fmt::{Display, Formatter};
use std::fmt::{Display, Formatter, LowerExp};
use std::str::FromStr;

use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc};
Expand Down Expand Up @@ -1210,11 +1210,66 @@ impl Datum {
/// Returns a human-readable string representation of this literal.
///
/// For string literals, this returns the raw string value without quotes.
/// Float and double literals are formatted like Java's `Float.toString` /
/// `Double.toString` (JDK 19+).
/// For all other literals, it falls back to [`to_string()`](ToString::to_string).
pub fn to_human_string(&self) -> String {
match self.literal() {
PrimitiveLiteral::String(s) => s.to_string(),
PrimitiveLiteral::Float(val) => human_float(val.0),
PrimitiveLiteral::Double(val) => human_float(val.0),
_ => self.to_string(),
}
}
}

/// Formats a float like Java's `Float.toString` / `Double.toString` (JDK 19+):
/// plain decimal when `1e-3 <= |value| < 1e7`, otherwise `<d>.<ddd>E<exp>`,
/// always with at least one digit after the point.
fn human_float<T: Float + LowerExp + FromStr>(value: T) -> String {
if value.is_nan() {
return "NaN".to_string();
}
let sign = if value.is_sign_negative() { "-" } else { "" };
if value.is_infinite() {
return format!("{sign}Infinity");
}
if value.is_zero() {
return format!("{sign}0.0");
}

let abs = value.abs();
// `{:e}` gives the fewest digits that round-trip. Java uses at least two
// digits and, among decimals of that length that round-trip, the closest
// one, preferring an even last digit on ties (2097152.2 for 2097152.25f32,
// 4.9E-324 for f64::from_bits(1)). Exact formatting picks exactly that.
let shortest = format!("{abs:e}");
let precision = shortest.find('e').unwrap().saturating_sub(2).max(1);
let closest = format!("{abs:.precision$e}");
let scientific = if closest.parse::<T>().is_ok_and(|v| v == abs) {
closest
} else {
shortest
};
let (mantissa, exponent) = scientific.split_once('e').unwrap();
let exponent: i32 = exponent.parse().unwrap();
let digits = mantissa.replace('.', "");
let digits = digits.trim_end_matches('0');

if (-3..7).contains(&exponent) {
if exponent < 0 {
let zeros = "0".repeat((-exponent - 1) as usize);
format!("{sign}0.{zeros}{digits}")
} else {
let int_len = exponent as usize + 1;
if digits.len() > int_len {
format!("{sign}{}.{}", &digits[..int_len], &digits[int_len..])
} else {
format!("{sign}{digits:0<int_len$}.0")
}
}
} else {
let fraction = if digits.len() > 1 { &digits[1..] } else { "0" };
format!("{sign}{}.{fraction}E{exponent}", &digits[..1])
}
}
35 changes: 35 additions & 0 deletions crates/iceberg/src/spec/values/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1986,3 +1986,38 @@ fn test_raw_literal_project_by_name_rejects_missing_required_field() {
let err = raw.project_by_name(&expected).unwrap_err();
assert_eq!(err.kind(), ErrorKind::DataInvalid);
}

/// Expected values are the output of Java's `Double.toString` / `Float.toString` (JDK 21).
#[test]
fn test_datum_to_human_string_float_matches_java() {
let double = |value: f64| Datum::double(value).to_human_string();
assert_eq!(double(1.0), "1.0");
assert_eq!(double(100.0), "100.0");
assert_eq!(double(0.0), "0.0");
assert_eq!(double(-0.0), "-0.0");
assert_eq!(double(1.5), "1.5");
assert_eq!(double(0.001), "0.001");
assert_eq!(double(9999999.0), "9999999.0");
assert_eq!(double(1.0e7), "1.0E7");
assert_eq!(double(1.0e-5), "1.0E-5");
assert_eq!(double(1.23456789e20), "1.23456789E20");
assert_eq!(double(1.0e23), "1.0E23");
assert_eq!(double(f64::INFINITY), "Infinity");
assert_eq!(double(f64::NEG_INFINITY), "-Infinity");
assert_eq!(double(f64::NAN), "NaN");
assert_eq!(double(f64::MAX), "1.7976931348623157E308");
// Java uses at least two significant digits.
assert_eq!(double(f64::from_bits(1)), "4.9E-324");
// Halfway between two 17-digit decimals: Java picks the even one.
assert_eq!(double(2f64.powi(-25)), "2.9802322387695312E-8");

let float = |value: f32| Datum::float(value).to_human_string();
assert_eq!(float(1.0), "1.0");
assert_eq!(float(-0.0), "-0.0");
assert_eq!(float(0.1), "0.1");
assert_eq!(float(1.0e7), "1.0E7");
assert_eq!(float(f32::NEG_INFINITY), "-Infinity");
assert_eq!(float(f32::from_bits(1)), "1.4E-45");
// 2097152.25, halfway between 2097152.2 and 2097152.3.
assert_eq!(float(f32::from_bits(0x4a00_0001)), "2097152.2");
}
Loading