diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index f3c9f9e54a..e7db641934 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -2005,6 +2005,41 @@ mod tests { ); } + /// The expected path is the output of iceberg-java's `PartitionSpec.partitionToPath`. + #[test] + fn test_partition_to_path_timestamp_and_binary_match_java() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "ts", Type::Primitive(PrimitiveType::Timestamp)).into(), + NestedField::required(2, "tstz", Type::Primitive(PrimitiveType::Timestamptz)) + .into(), + NestedField::required(3, "b", Type::Primitive(PrimitiveType::Binary)).into(), + ]) + .build() + .unwrap(); + + let spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("ts", "ts", Transform::Identity) + .unwrap() + .add_partition_field("tstz", "tstz", Transform::Identity) + .unwrap() + .add_partition_field("b", "b", Transform::Identity) + .unwrap() + .build() + .unwrap(); + + let data = Struct::from_iter([ + Some(Literal::timestamp(1510871468123456)), + Some(Literal::timestamptz(1510871468123456)), + Some(Literal::binary(vec![0x01, 0x02, 0xFF])), + ]); + + assert_eq!( + spec.partition_to_path(&data, schema.into()), + "ts=2017-11-16T22%3A31%3A08.123456/tstz=2017-11-16T22%3A31%3A08.123456%2B00%3A00/b=AQL%2F" + ); + } + #[test] fn test_partition_to_path_escaped_field_name() { let schema = Schema::builder() diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index c28dc28c19..6ff1b150ea 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -1306,6 +1306,85 @@ mod tests { ); } + fn human_identity(primitive: PrimitiveType, literal: Literal) -> String { + Transform::Identity.to_human_string(&Type::Primitive(primitive), Some(&literal)) + } + + /// Expected values are the output of iceberg-java's `Transform.toHumanString`. + #[test] + fn test_to_human_string_timestamp_matches_java() { + let cases = [ + (1510871468123456, "2017-11-16T22:31:08.123456"), + (1510871460000000, "2017-11-16T22:31:00"), + (1510871468000000, "2017-11-16T22:31:08"), + (1510871460120000, "2017-11-16T22:31:00.12"), + (1510871460100000, "2017-11-16T22:31:00.1"), + (1510871460000001, "2017-11-16T22:31:00.000001"), + (0, "1970-01-01T00:00:00"), + (-1, "1969-12-31T23:59:59.999999"), + (-1000000, "1969-12-31T23:59:59"), + ]; + for (micros, expected) in cases { + assert_eq!( + human_identity(PrimitiveType::Timestamp, Literal::timestamp(micros)), + expected + ); + assert_eq!( + human_identity(PrimitiveType::Timestamptz, Literal::timestamptz(micros)), + format!("{expected}+00:00") + ); + } + } + + /// Expected values are the output of iceberg-java's `Transform.toHumanString`. + #[test] + fn test_to_human_string_timestamp_nanos_matches_java() { + let cases = [ + (1510871468123456789, "2017-11-16T22:31:08.123456789"), + (1510871460000000000, "2017-11-16T22:31:00"), + (1510871460000000001, "2017-11-16T22:31:00.000000001"), + (1510871460120000000, "2017-11-16T22:31:00.12"), + ]; + for (nanos, expected) in cases { + assert_eq!( + human_identity(PrimitiveType::TimestampNs, Literal::timestamp_nano(nanos)), + expected + ); + assert_eq!( + human_identity( + PrimitiveType::TimestamptzNs, + Literal::timestamptz_nano(nanos) + ), + format!("{expected}+00:00") + ); + } + } + + /// Expected values are the output of iceberg-java's `Transform.toHumanString`. + #[test] + fn test_to_human_string_binary_matches_java() { + let cases: [(&[u8], &str); 5] = [ + (&[], ""), + (&[0xFB], "+w=="), + (&[0xFB, 0xFF], "+/8="), + (&[0x01, 0x02, 0xFF], "AQL/"), + (b"hello", "aGVsbG8="), + ]; + for (bytes, expected) in cases { + assert_eq!( + human_identity(PrimitiveType::Binary, Literal::binary(bytes.to_vec())), + expected + ); + } + assert_eq!( + human_identity( + PrimitiveType::Fixed(3), + Literal::fixed(vec![0x01, 0x02, 0xFF]) + ), + "AQL/" + ); + } + #[test] fn test_to_human_string_null_cases() { assert_eq!(human(Transform::Void, PrimitiveType::Int, 47), "null"); diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index a501466b39..70cf76fe9a 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -21,6 +21,8 @@ use std::cmp::Ordering; use std::fmt::{Display, Formatter, LowerExp}; use std::str::FromStr; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; use ordered_float::{Float, OrderedFloat}; use serde::de::{self, MapAccess}; @@ -1225,15 +1227,37 @@ impl Datum { /// Returns a human-readable string representation of this literal. /// - /// For string literals, this returns the raw string value without quotes. - /// Float and double literals are formatted like Java's `Float.toString` / - /// `Double.toString` (JDK 19+). - /// For all other literals, it falls back to [`to_string()`](ToString::to_string). + /// This is the value written into partition paths, so it mirrors the output of + /// iceberg-java's `Transform.toHumanString`: + /// - string literals are returned without quotes; + /// - float and double literals are formatted like Java's `Float.toString` / + /// `Double.toString` (JDK 19+); + /// - timestamps use `yyyy-MM-ddTHH:mm:ss` with trailing zeros of the fraction trimmed, + /// followed by `+00:00` when the type has a zone (`TransformUtil.humanTimestamp*`); + /// - binary and fixed values are standard Base64 with padding + /// (`TransformUtil.base64encode`). + /// + /// All other literals fall back to [`to_string()`](ToString::to_string). pub fn to_human_string(&self) -> String { - match self.literal() { - PrimitiveLiteral::String(s) => s.to_string(), - PrimitiveLiteral::Float(val) => human_float(val.0), - PrimitiveLiteral::Double(val) => human_float(val.0), + match (&self.r#type, self.literal()) { + (_, PrimitiveLiteral::String(s)) => s.to_string(), + (_, PrimitiveLiteral::Float(val)) => human_float(val.0), + (_, PrimitiveLiteral::Double(val)) => human_float(val.0), + (PrimitiveType::Timestamp, PrimitiveLiteral::Long(micros)) => { + human_timestamp(timestamp::microseconds_to_datetime(*micros)) + } + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(nanos)) => { + human_timestamp(timestamp::nanoseconds_to_datetime(*nanos)) + } + (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(micros)) => { + let datetime = timestamptz::microseconds_to_datetimetz(*micros).naive_utc(); + format!("{}+00:00", human_timestamp(datetime)) + } + (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(nanos)) => { + let datetime = timestamptz::nanoseconds_to_datetimetz(*nanos).naive_utc(); + format!("{}+00:00", human_timestamp(datetime)) + } + (_, PrimitiveLiteral::Binary(bytes)) => BASE64.encode(bytes), _ => self.to_string(), } } @@ -1289,3 +1313,18 @@ fn human_float(value: T) -> String { format!("{sign}{}.{fraction}E{exponent}", &digits[..1]) } } + +/// Formats a timestamp as `yyyy-MM-ddTHH:mm:ss`, then the fraction of a second with its +/// trailing zeros trimmed, so `.120000` is written as `.12` and a whole second has no +/// fraction. +/// +/// Mirrors the output of `TransformUtil.humanTimestampWithoutZone`. +fn human_timestamp(datetime: NaiveDateTime) -> String { + let mut human = datetime.format("%Y-%m-%dT%H:%M:%S").to_string(); + let nanos = datetime.and_utc().timestamp_subsec_nanos(); + if nanos > 0 { + human.push('.'); + human.push_str(format!("{nanos:09}").trim_end_matches('0')); + } + human +}