From 8030cf44445b3f957e367515e9d3c5eeb00f104d Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Thu, 17 Sep 2026 08:16:13 +0000 Subject: [PATCH 01/63] feat(spec): add unknown primitive type support (#2773) * feat(spec): add unknown primitive type support Co-authored-by: Codex * fix(arrow): validate row ID synthesis inputs * fix(spec): centralize unknown field validation * fix(spec): address unknown type review feedback * fix(spec): address remaining review feedback * fix(spec): centralize dropped partition source handling * fix(inspect): recover historical partition bounds * fix(spec): validate default partition sources on metadata load * Revert "fix(inspect): recover historical partition bounds" This reverts commit 6403d4898edd03b433c64965b1ed7195686a0330. --------- Co-authored-by: Codex --- crates/catalog/glue/src/schema.rs | 6 + crates/catalog/hms/src/schema.rs | 6 + crates/iceberg/public-api.txt | 1 + crates/iceberg/src/arrow/schema.rs | 11 ++ crates/iceberg/src/arrow/value.rs | 24 +++- crates/iceberg/src/avro/schema.rs | 106 +++++++++++--- crates/iceberg/src/inspect/manifests.rs | 61 ++++++--- crates/iceberg/src/scan/mod.rs | 35 +++-- crates/iceberg/src/scan/task.rs | 9 +- crates/iceberg/src/spec/datatypes.rs | 24 +++- crates/iceberg/src/spec/partition.rs | 124 +++++++++++++---- crates/iceberg/src/spec/schema/mod.rs | 160 ++++++++++++++++++++++ crates/iceberg/src/spec/table_metadata.rs | 86 +++++++++++- crates/iceberg/src/spec/values/datum.rs | 6 + crates/iceberg/src/spec/values/literal.rs | 27 ++-- crates/iceberg/src/spec/values/tests.rs | 17 +++ 16 files changed, 599 insertions(+), 104 deletions(-) diff --git a/crates/catalog/glue/src/schema.rs b/crates/catalog/glue/src/schema.rs index 6a33a9cba2..6116fc224b 100644 --- a/crates/catalog/glue/src/schema.rs +++ b/crates/catalog/glue/src/schema.rs @@ -149,6 +149,12 @@ impl SchemaVisitor for GlueSchemaBuilder { fn primitive(&mut self, p: &PrimitiveType) -> Result { let glue_type = match p { + PrimitiveType::Unknown => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Conversion from {p:?} is not supported"), + )); + } PrimitiveType::Boolean => "boolean".to_string(), PrimitiveType::Int => "int".to_string(), PrimitiveType::Long => "bigint".to_string(), diff --git a/crates/catalog/hms/src/schema.rs b/crates/catalog/hms/src/schema.rs index e21a75f976..f04ee11c17 100644 --- a/crates/catalog/hms/src/schema.rs +++ b/crates/catalog/hms/src/schema.rs @@ -100,6 +100,12 @@ impl SchemaVisitor for HiveSchemaBuilder { fn primitive(&mut self, p: &PrimitiveType) -> Result { let hive_type = match p { + PrimitiveType::Unknown => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Conversion from {p:?} is not supported"), + )); + } PrimitiveType::Boolean => "boolean".to_string(), PrimitiveType::Int => "int".to_string(), PrimitiveType::Long => "bigint".to_string(), diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 7d03d6b1fb..a917d28f5a 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1664,6 +1664,7 @@ pub iceberg::spec::PrimitiveType::Timestamp pub iceberg::spec::PrimitiveType::TimestampNs pub iceberg::spec::PrimitiveType::Timestamptz pub iceberg::spec::PrimitiveType::TimestamptzNs +pub iceberg::spec::PrimitiveType::Unknown pub iceberg::spec::PrimitiveType::Uuid impl iceberg::spec::PrimitiveType pub fn iceberg::spec::PrimitiveType::compatible(&self, literal: &iceberg::spec::PrimitiveLiteral) -> bool diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 349ada859e..4ef7b29436 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -191,6 +191,7 @@ fn visit_type(r#type: &DataType, visitor: &mut V) -> Resu | DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + | DataType::Null | DataType::Binary | DataType::LargeBinary | DataType::BinaryView @@ -509,6 +510,7 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { fn primitive(&mut self, p: &DataType) -> Result { match p { + DataType::Null => Ok(Type::Primitive(PrimitiveType::Unknown)), DataType::Boolean => Ok(Type::Primitive(PrimitiveType::Boolean)), DataType::Int8 | DataType::Int16 | DataType::Int32 => { Ok(Type::Primitive(PrimitiveType::Int)) @@ -697,6 +699,7 @@ impl SchemaVisitor for ToArrowSchemaConverter { fn primitive(&mut self, p: &PrimitiveType) -> Result { match p { + PrimitiveType::Unknown => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Null)), PrimitiveType::Boolean => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Boolean)), PrimitiveType::Int => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int32)), PrimitiveType::Long => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int64)), @@ -1209,6 +1212,7 @@ pub(crate) fn primitive_type_to_arrow_type_with_ree(primitive_type: &PrimitiveTy }; match primitive_type { + PrimitiveType::Unknown => make_ree(DataType::Null), PrimitiveType::Boolean => make_ree(DataType::Boolean), PrimitiveType::Int => make_ree(DataType::Int32), PrimitiveType::Long => make_ree(DataType::Int64), @@ -2227,6 +2231,13 @@ mod tests { assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap()); } + { + let arrow_type = DataType::Null; + let iceberg_type = Type::Primitive(PrimitiveType::Unknown); + assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap()); + assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap()); + } + // test struct type { // no metadata will cause error diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index ff28052fce..ad2a7f6476 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -206,6 +206,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { fn primitive(&mut self, p: &PrimitiveType, partner: &ArrayRef) -> Result>> { match p { + PrimitiveType::Unknown => Ok(vec![None; partner.len()]), PrimitiveType::Boolean => { let array = partner .as_any() @@ -636,6 +637,7 @@ pub(crate) fn create_primitive_array_single_element( prim_lit: Option<&PrimitiveLiteral>, ) -> Result { match (data_type, prim_lit) { + (DataType::Null, None) => Ok(Arc::new(arrow_array::NullArray::new(1))), (DataType::Boolean, Some(PrimitiveLiteral::Boolean(v))) => { Ok(Arc::new(BooleanArray::from(vec![*v]))) } @@ -963,7 +965,7 @@ pub(crate) fn create_primitive_array_repeated( Some(NullBuffer::new_null(num_rows)), )) } - (DataType::Null, _) => Arc::new(arrow_array::NullArray::new(num_rows)), + (DataType::Null, None) => Arc::new(arrow_array::NullArray::new(num_rows)), // --- Catch-all null arm: use arrow-rs new_null_array for any remaining DataType --- (dt, None) => new_null_array(dt, num_rows), @@ -1886,6 +1888,26 @@ mod test { } } + #[test] + fn test_create_null_array_rejects_non_null_literal() { + let literal = Some(PrimitiveLiteral::Int(1)); + + assert!(create_primitive_array_single_element(&DataType::Null, literal.as_ref()).is_err()); + assert!(create_primitive_array_repeated(&DataType::Null, literal.as_ref(), 2).is_err()); + assert_eq!( + create_primitive_array_single_element(&DataType::Null, None) + .unwrap() + .len(), + 1 + ); + assert_eq!( + create_primitive_array_repeated(&DataType::Null, None, 2) + .unwrap() + .len(), + 2 + ); + } + #[test] fn test_create_decimal_array_repeated_respects_precision() { // Ensure repeated arrays also respect target precision, not Arrow's default. diff --git a/crates/iceberg/src/avro/schema.rs b/crates/iceberg/src/avro/schema.rs index 528b89b7c1..a67e4ed71b 100644 --- a/crates/iceberg/src/avro/schema.rs +++ b/crates/iceberg/src/avro/schema.rs @@ -75,7 +75,13 @@ impl SchemaVisitor for SchemaToAvroSchema { } if !field.required { - field_schema = avro_optional(field_schema)?; + if is_avro_null(&field_schema) { + // Unknown maps directly to Avro null. Wrapping it as optional would create + // an invalid union with duplicate null variants. + field_schema = AvroSchema::Null; + } else { + field_schema = avro_optional(field_schema)?; + } } let default = if let Some(literal) = &field.initial_default { @@ -127,7 +133,13 @@ impl SchemaVisitor for SchemaToAvroSchema { } if !list.element_field.required { - field_schema = avro_optional(field_schema)?; + if is_avro_null(&field_schema) { + // Unknown maps directly to Avro null. Wrapping it as optional would create + // an invalid union with duplicate null variants. + field_schema = AvroSchema::Null; + } else { + field_schema = avro_optional(field_schema)?; + } } Ok(Either::Left(AvroSchema::Array(ArraySchema { @@ -148,7 +160,13 @@ impl SchemaVisitor for SchemaToAvroSchema { let key_field_schema = key_value.unwrap_left(); let mut value_field_schema = value.unwrap_left(); if !map.value_field.required { - value_field_schema = avro_optional(value_field_schema)?; + if is_avro_null(&value_field_schema) { + // Unknown maps directly to Avro null. Wrapping it as optional would create + // an invalid union with duplicate null variants. + value_field_schema = AvroSchema::Null; + } else { + value_field_schema = avro_optional(value_field_schema)?; + } } if matches!(key_field_schema, AvroSchema::String) { @@ -222,6 +240,7 @@ impl SchemaVisitor for SchemaToAvroSchema { fn primitive(&mut self, p: &PrimitiveType) -> Result { let avro_schema = match p { + PrimitiveType::Unknown => AvroSchema::Null, PrimitiveType::Boolean => AvroSchema::Boolean, PrimitiveType::Int => AvroSchema::Int, PrimitiveType::Long => AvroSchema::Long, @@ -324,6 +343,14 @@ fn is_avro_optional(avro_schema: &AvroSchema) -> bool { } } +fn is_avro_null(avro_schema: &AvroSchema) -> bool { + matches!(avro_schema, AvroSchema::Null) +} + +fn unwrap_or_unknown(field_type: Option) -> Type { + field_type.unwrap_or(Type::Primitive(PrimitiveType::Unknown)) +} + /// Post order avro schema visitor. pub(crate) trait AvroSchemaVisitor { type T; @@ -447,10 +474,10 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { let field_id = Self::get_element_id_from_attributes(&avro_field.custom_attributes, FIELD_ID_PROP)?; - let optional = is_avro_optional(&avro_field.schema); + let optional = is_avro_optional(&avro_field.schema) || is_avro_null(&avro_field.schema); - let mut field = - NestedField::new(field_id, &avro_field.name, field_type.unwrap(), !optional); + let field_type = unwrap_or_unknown(field_type); + let mut field = NestedField::new(field_id, &avro_field.name, field_type, !optional); if let Some(doc) = &avro_field.doc { field = field.with_doc(doc); @@ -482,7 +509,7 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { } if options.len() == 1 { - Ok(Some(options.remove(0).unwrap())) + Ok(Some(unwrap_or_unknown(options.remove(0)))) } else { Ok(Some(options.remove(1).unwrap())) } @@ -490,10 +517,11 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { fn array(&mut self, array: &ArraySchema, item: Option) -> Result { let element_field_id = Self::get_element_id_from_attributes(&array.attributes, ELEMENT_ID)?; + let item = unwrap_or_unknown(item); let element_field = NestedField::list_element( element_field_id, - item.unwrap(), - !is_avro_optional(&array.items), + item, + !is_avro_optional(&array.items) && !is_avro_null(&array.items), ) .into(); Ok(Some(Type::List(ListType { element_field }))) @@ -504,10 +532,11 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { let key_field = NestedField::map_key_element(key_field_id, Type::Primitive(PrimitiveType::String)); let value_field_id = Self::get_element_id_from_attributes(&map.attributes, VALUE_ID)?; + let value = unwrap_or_unknown(value); let value_field = NestedField::map_value_element( value_field_id, - value.unwrap(), - !is_avro_optional(&map.types), + value, + !is_avro_optional(&map.types) && !is_avro_null(&map.types), ); Ok(Some(Type::Map(MapType { key_field: key_field.into(), @@ -557,12 +586,7 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { "Can't convert avro map schema, missing key schema.", ) })?; - let value = value.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro map schema, missing value schema.", - ) - })?; + let value = unwrap_or_unknown(value); let key_id = Self::get_element_id_from_attributes( &array.fields[0].custom_attributes, FIELD_ID_PROP, @@ -575,7 +599,7 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { let value_field = NestedField::map_value_element( value_id, value, - !is_avro_optional(&array.fields[1].schema), + !is_avro_optional(&array.fields[1].schema) && !is_avro_null(&array.fields[1].schema), ); Ok(Some(Type::Map(MapType { key_field: key_field.into(), @@ -659,6 +683,52 @@ mod tests { assert_eq!(iceberg_schema, converted_avro_converted_iceberg_schema); } + #[test] + fn test_unknown_type_schema_conversion() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "empty", PrimitiveType::Unknown.into()).into(), + NestedField::required( + 2, + "unknowns", + Type::List(ListType::new( + NestedField::list_element(3, PrimitiveType::Unknown.into(), false).into(), + )), + ) + .into(), + NestedField::required( + 4, + "unknown_values", + Type::Map(MapType::optional( + 5, + PrimitiveType::String.into(), + 6, + PrimitiveType::Unknown.into(), + )), + ) + .into(), + ]) + .build() + .unwrap(); + + let avro_schema = schema_to_avro_schema("table", &schema).unwrap(); + let AvroSchema::Record(record) = &avro_schema else { + panic!("expected avro record schema"); + }; + assert!(is_avro_null(&record.fields[0].schema)); + assert_eq!(record.fields[0].default, Some(Value::Null)); + let AvroSchema::Array(array) = &record.fields[1].schema else { + panic!("expected Avro array schema"); + }; + assert!(is_avro_null(&array.items)); + let AvroSchema::Map(map) = &record.fields[2].schema else { + panic!("expected Avro map schema"); + }; + assert!(is_avro_null(&map.types)); + + assert_eq!(schema, avro_schema_to_schema(&avro_schema).unwrap()); + } + #[test] fn test_manifest_file_v1_schema() { let fields = vec![ diff --git a/crates/iceberg/src/inspect/manifests.rs b/crates/iceberg/src/inspect/manifests.rs index 38351a8c54..9e0f7f2240 100644 --- a/crates/iceberg/src/inspect/manifests.rs +++ b/crates/iceberg/src/inspect/manifests.rs @@ -260,22 +260,22 @@ impl<'a> ManifestsTable<'a> { .unwrap() .append_option(summary.contains_nan); - partition_summaries_builder - .field_builder::(2) - .unwrap() - .append_option(summary.lower_bound.as_ref().map(|v| { - Datum::try_from_bytes(v, field.field_type.as_primitive_type().unwrap().clone()) - .unwrap() - .to_string() - })); - partition_summaries_builder - .field_builder::(3) - .unwrap() - .append_option(summary.upper_bound.as_ref().map(|v| { - Datum::try_from_bytes(v, field.field_type.as_primitive_type().unwrap().clone()) - .unwrap() - .to_string() - })); + let field_type = field.field_type.as_primitive_type().unwrap(); + for (index, bound) in [(2, &summary.lower_bound), (3, &summary.upper_bound)] { + // Bounds cannot be decoded when the source column's type is unknown. + let bound = bound + .as_ref() + .filter(|_| *field_type != PrimitiveType::Unknown) + .map(|v| { + Datum::try_from_bytes(v, field_type.clone()) + .unwrap() + .to_string() + }); + partition_summaries_builder + .field_builder::(index) + .unwrap() + .append_option(bound); + } partition_summaries_builder.append(true); } builder.append(true); @@ -286,6 +286,7 @@ impl<'a> ManifestsTable<'a> { mod tests { use std::sync::Arc; + use arrow_array::{Array, ListArray, StructArray}; use expect_test::expect; use futures::TryStreamExt; @@ -418,9 +419,29 @@ mod tests { let metadata: TableMetadata = serde_json::from_value(metadata).unwrap(); let table = fixture.table.clone().with_metadata(Arc::new(metadata)); - match table.inspect().manifests().scan().await { - Err(err) => assert!(err.to_string().contains("No column with source column id")), - Ok(_) => panic!("expected scan to fail for a dropped partition source column"), - } + let batches: Vec<_> = table + .inspect() + .manifests() + .scan() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1); + let summaries = batches[0] + .column_by_name("partition_summaries") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let summary = summaries.value(0); + let summary = summary.as_any().downcast_ref::().unwrap(); + assert_eq!(summary.len(), 1); + assert!(summary.column_by_name("contains_null").unwrap().is_valid(0)); + assert!(summary.column_by_name("contains_nan").unwrap().is_valid(0)); + assert!(summary.column_by_name("lower_bound").unwrap().is_null(0)); + assert!(summary.column_by_name("upper_bound").unwrap().is_null(0)); } } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 081d64273d..fd833d9b38 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -2284,10 +2284,15 @@ pub mod tests { } #[tokio::test] - async fn test_filtered_scan_rejects_dropped_partition_source_column() { + async fn test_filtered_scan_with_dropped_partition_source_column() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; + // Baseline: the same filtered scan against the table before evolution. + let baseline = scan_y_gte_5(&fixture.table).await; + assert!(!baseline.is_empty()); + assert!(baseline.iter().all(|y| *y >= 5)); + // Evolve the table so that the manifests reference a historical spec whose source // column is no longer in the current schema: make an unpartitioned spec the // default, then drop the original spec's source column from the schema. @@ -2337,23 +2342,37 @@ pub mod tests { .metadata; let table = fixture.table.clone().with_metadata(Arc::new(metadata)); + // Planning and reading must succeed, and the results must match the table before + // evolution: no rows wrongly pruned and none returned unfiltered. + let evolved = scan_y_gte_5(&table).await; + assert_eq!(evolved, baseline); + } + + async fn scan_y_gte_5(table: &Table) -> Vec { let table_scan = table .scan() .select(["y"]) .with_filter(Reference::new("y").greater_than_or_equal_to(Datum::long(5))) .build() .unwrap(); - - let err = table_scan - .plan_files() + let batches: Vec<_> = table_scan + .to_arrow() .await .unwrap() - .try_collect::>() + .try_collect() .await - .unwrap_err(); + .unwrap(); - assert_eq!(err.kind(), ErrorKind::Unexpected); - assert!(err.message().contains("No column with source column id 1")); + let mut values: Vec = batches + .iter() + .flat_map(|batch| { + let col = batch.column_by_name("y").unwrap(); + let arr = col.as_any().downcast_ref::().unwrap(); + (0..arr.len()).map(|i| arr.value(i)).collect::>() + }) + .collect(); + values.sort_unstable(); + values } #[tokio::test] diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 5fd0f81bb6..531f405567 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -689,7 +689,7 @@ mod tests { } #[test] - fn test_file_scan_task_builder_rejects_dropped_partition_source_column() { + fn test_file_scan_task_builder_accepts_dropped_partition_source_column() { let (_historical_schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity); let current_schema = Arc::new( @@ -703,15 +703,12 @@ mod tests { .unwrap(), ); - let err = build_file_scan_task( + build_file_scan_task( current_schema, Some(Struct::from_iter([Some(Literal::long(42))])), Some(partition_spec), ) - .unwrap_err(); - - assert_eq!(err.kind(), ErrorKind::Unexpected); - assert!(err.message().contains("No column with source column id 1")); + .unwrap(); } #[test] diff --git a/crates/iceberg/src/spec/datatypes.rs b/crates/iceberg/src/spec/datatypes.rs index e53f856c00..8c4029f7b3 100644 --- a/crates/iceberg/src/spec/datatypes.rs +++ b/crates/iceberg/src/spec/datatypes.rs @@ -133,8 +133,8 @@ impl Type { /// Minimum [`FormatVersion`] required to support this type, **without** taking /// nested field types into account. /// - /// `TimestampNs` / `TimestamptzNs` / `Variant` require [`FormatVersion::V3`]; every - /// other type is valid from [`FormatVersion::V1`]. Mirrors Java's + /// `Unknown` / `TimestampNs` / `TimestamptzNs` / `Variant` require + /// [`FormatVersion::V3`]; every other type is valid from [`FormatVersion::V1`]. Mirrors Java's /// `Schema.MIN_FORMAT_VERSIONS` (a shallow lookup keyed by type id), so it /// intentionally does not recurse: callers needing the floor for a whole schema /// iterate its flattened fields (see [`Schema::calc_min_compatible_format`]). @@ -142,7 +142,9 @@ impl Type { /// [`Schema::calc_min_compatible_format`]: crate::spec::Schema::calc_min_compatible_format pub(crate) fn min_format_version(&self) -> FormatVersion { match self { - Type::Primitive(PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs) + Type::Primitive( + PrimitiveType::Unknown | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs, + ) | Type::Variant(_) => FormatVersion::V3, _ => FormatVersion::V1, } @@ -273,6 +275,8 @@ pub enum PrimitiveType { Fixed(u64), /// Arbitrary-length byte array. Binary, + /// Default / null column type used when a more specific type is not known. + Unknown, } impl PrimitiveType { @@ -390,6 +394,7 @@ where S: Serializer { impl fmt::Display for PrimitiveType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { + PrimitiveType::Unknown => write!(f, "unknown"), PrimitiveType::Boolean => write!(f, "boolean"), PrimitiveType::Int => write!(f, "int"), PrimitiveType::Long => write!(f, "long"), @@ -596,12 +601,12 @@ impl TryFrom for NestedField { fn try_from(value: SerdeNestedField) -> Result { let initial_default = value .initial_default - .map(|x| Literal::try_from_json(x, &value.field_type)) + .map(|default| Literal::try_from_json(default, &value.field_type)) .transpose()? .flatten(); let write_default = value .write_default - .map(|x| Literal::try_from_json(x, &value.field_type)) + .map(|default| Literal::try_from_json(default, &value.field_type)) .transpose()? .flatten(); @@ -940,6 +945,7 @@ mod tests { { "type": "struct", "fields": [ + {"id": 17, "name": "unknown_field", "required": false, "type": "unknown"}, {"id": 1, "name": "bool_field", "required": true, "type": "boolean"}, {"id": 2, "name": "int_field", "required": true, "type": "int"}, {"id": 3, "name": "long_field", "required": true, "type": "long"}, @@ -964,6 +970,12 @@ mod tests { record, Type::Struct(StructType { fields: vec![ + NestedField::optional( + 17, + "unknown_field", + Type::Primitive(PrimitiveType::Unknown), + ) + .into(), NestedField::required(1, "bool_field", Type::Primitive(PrimitiveType::Boolean)) .into(), NestedField::required(2, "int_field", Type::Primitive(PrimitiveType::Int)) @@ -1345,6 +1357,8 @@ mod tests { for (ty, literal) in pairs { assert!(ty.compatible(&literal)); } + + assert!(!PrimitiveType::Unknown.compatible(&PrimitiveLiteral::Int(1))); } #[test] diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index 0fe5739234..c79eaae639 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -25,7 +25,7 @@ use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use super::transform::Transform; -use super::{NestedField, Schema, SchemaRef, StructType}; +use super::{NestedField, PrimitiveType, Schema, SchemaRef, StructType}; use crate::spec::Struct; use crate::{Error, ErrorKind, Result}; @@ -103,8 +103,32 @@ impl PartitionSpec { } /// Returns the partition type of this partition spec. + /// If a source column is absent, preserves fixed transform result types and uses + /// unknown for result types that depend on the source type. pub fn partition_type(&self, schema: &Schema) -> Result { - PartitionSpecBuilder::partition_type(&self.fields, schema) + let mut struct_fields = Vec::with_capacity(self.fields.len()); + for partition_field in &self.fields { + let res_type = match schema.field_by_id(partition_field.source_id) { + Some(field) => partition_field.transform.result_type(&field.field_type)?, + // Historical specs may reference dropped source columns. Retain every + // field's position and any result type that is independent of its source. + None => match partition_field.transform { + Transform::Bucket(_) | Transform::Year | Transform::Month | Transform::Hour => { + PrimitiveType::Int.into() + } + Transform::Day => PrimitiveType::Date.into(), + Transform::Unknown => PrimitiveType::String.into(), + Transform::Identity | Transform::Truncate(_) | Transform::Void => { + PrimitiveType::Unknown.into() + } + }, + }; + struct_fields.push( + NestedField::optional(partition_field.field_id, &partition_field.name, res_type) + .into(), + ); + } + Ok(StructType::new(struct_fields)) } /// Convert to unbound partition spec @@ -566,32 +590,6 @@ impl PartitionSpecBuilder { Ok(bound_fields) } - /// Returns the partition type of this partition spec. - fn partition_type(fields: &Vec, schema: &Schema) -> Result { - let mut struct_fields = Vec::with_capacity(fields.len()); - for partition_field in fields { - let field = schema - .field_by_id(partition_field.source_id) - .ok_or_else(|| { - Error::new( - // This should never occur as check_transform_compatibility - // already ensures that the source field exists in the schema - ErrorKind::Unexpected, - format!( - "No column with source column id {} in schema {:?}", - partition_field.source_id, schema - ), - ) - })?; - let res_type = partition_field.transform.result_type(&field.field_type)?; - let field = - NestedField::optional(partition_field.field_id, &partition_field.name, res_type) - .into(); - struct_fields.push(field); - } - Ok(StructType::new(struct_fields)) - } - /// Ensure that the partition name is unique among columns in the schema. /// Duplicate names are allowed if: /// 1. The column is sourced from the column with the same name. @@ -1055,7 +1053,7 @@ mod tests { } #[test] - fn test_partition_error() { + fn test_partition_type_with_dropped_source_column() { let spec = r#" { "spec-id": 1, @@ -1087,7 +1085,73 @@ mod tests { .build() .unwrap(); - assert!(partition_spec.partition_type(&schema).is_err()); + assert_eq!( + partition_spec.partition_type(&schema).unwrap(), + StructType::new(vec![ + NestedField::optional(1000, "ts_day", PrimitiveType::Date.into()).into(), + NestedField::optional(1001, "id_bucket", PrimitiveType::Int.into()).into(), + NestedField::optional(1002, "id_truncate", PrimitiveType::String.into()).into(), + ]) + ); + + // Dropping all sources must retain every partition field in its original position. + let empty_schema = Schema::builder().build().unwrap(); + assert_eq!( + partition_spec.partition_type(&empty_schema).unwrap(), + StructType::new(vec![ + NestedField::optional(1000, "ts_day", PrimitiveType::Date.into()).into(), + NestedField::optional(1001, "id_bucket", PrimitiveType::Int.into()).into(), + NestedField::optional(1002, "id_truncate", PrimitiveType::Unknown.into()).into(), + ]) + ); + + // Missing sources must not suppress validation of transforms on remaining sources. + let incompatible_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", PrimitiveType::Boolean.into()).into(), + ]) + .build() + .unwrap(); + let err = partition_spec + .partition_type(&incompatible_schema) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "boolean is not a valid input type of bucket transform" + ); + } + + #[test] + fn test_partition_type_without_source_type() { + let schema = Schema::builder().build().unwrap(); + for (transform, expected_type) in [ + (Transform::Identity, PrimitiveType::Unknown), + (Transform::Truncate(4), PrimitiveType::Unknown), + (Transform::Void, PrimitiveType::Unknown), + (Transform::Bucket(16), PrimitiveType::Int), + (Transform::Year, PrimitiveType::Int), + (Transform::Month, PrimitiveType::Int), + (Transform::Day, PrimitiveType::Date), + (Transform::Hour, PrimitiveType::Int), + (Transform::Unknown, PrimitiveType::String), + ] { + let spec = PartitionSpec { + spec_id: 0, + fields: vec![PartitionField { + source_id: 1, + field_id: 1000, + name: "partition".to_string(), + transform, + }], + }; + assert_eq!( + spec.partition_type(&schema).unwrap(), + StructType::new(vec![ + NestedField::optional(1000, "partition", expected_type.into()).into(), + ]) + ); + } } #[test] diff --git a/crates/iceberg/src/spec/schema/mod.rs b/crates/iceberg/src/spec/schema/mod.rs index 35af6f0f84..a0197b82c9 100644 --- a/crates/iceberg/src/spec/schema/mod.rs +++ b/crates/iceberg/src/spec/schema/mod.rs @@ -647,6 +647,17 @@ mod tests { ]); assert_eq!(variant.calc_min_compatible_format(), FormatVersion::V3); + // Unknown is a v3-only primitive type. + let unknown = schema_with(vec![ + NestedField::optional(1, "u", Primitive(PrimitiveType::Unknown)).into(), + ]); + assert_eq!(unknown.calc_min_compatible_format(), FormatVersion::V3); + assert!( + unknown + .check_format_compatibility(FormatVersion::V2) + .is_err() + ); + // A v3-only type nested inside a list inside a struct → V3 (flattened fields). let nested = schema_with(vec![ NestedField::required( @@ -1475,4 +1486,153 @@ table { .is_err() ); } + + #[test] + fn test_unknown_type_deserialization_rejects_non_null_default() { + let field_json = serde_json::json!({ + "id": 1, + "name": "empty", + "required": false, + "type": "unknown", + "initial-default": 1 + }); + + let error = serde_json::from_value::(field_json.clone()).unwrap_err(); + assert!( + error + .to_string() + .contains("Unknown type only supports null default values"), + "unexpected error: {error}" + ); + + let schema_json = serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [field_json] + }); + assert!(serde_json::from_value::(schema_json).is_err()); + } + + #[test] + fn test_unknown_type_deserialization_accepts_null_defaults() { + let schema_json = serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [ + { + "id": 1, + "name": "empty", + "required": false, + "type": "unknown", + "initial-default": null, + "write-default": null + } + ] + }); + + serde_json::from_value::(schema_json).unwrap(); + } + + #[test] + fn test_unknown_type_accepts_null_container_defaults() { + let cases = [ + ( + "struct", + Struct(StructType::new(vec![ + NestedField::optional(2, "empty", Primitive(PrimitiveType::Unknown)).into(), + ])), + Literal::Struct(crate::spec::Struct::from_iter([None])), + ), + ( + "list", + List(ListType::new( + NestedField::list_element(2, Primitive(PrimitiveType::Unknown), false).into(), + )), + Literal::List(vec![None]), + ), + ( + "map", + Map(MapType::optional( + 2, + Primitive(PrimitiveType::String), + 3, + Primitive(PrimitiveType::Unknown), + )), + Literal::Map(MapValue::from([(Literal::string("key"), None)])), + ), + ]; + + for (name, field_type, default) in cases { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, name, field_type) + .with_write_default(default) + .into(), + ]) + .build() + .unwrap(); + serde_json::to_value(schema).unwrap(); + } + } + + #[test] + fn test_unknown_type_deserialization_rejects_non_null_container_defaults() { + let cases = [ + ( + "struct", + serde_json::json!({ + "type": "struct", + "fields": [{ + "id": 2, + "name": "empty", + "required": false, + "type": "unknown" + }] + }), + serde_json::json!({"2": 1}), + ), + ( + "list", + serde_json::json!({ + "type": "list", + "element-id": 2, + "element-required": false, + "element": "unknown" + }), + serde_json::json!([1]), + ), + ( + "map", + serde_json::json!({ + "type": "map", + "key-id": 2, + "key": "string", + "value-id": 3, + "value-required": false, + "value": "unknown" + }), + serde_json::json!({"keys": ["key"], "values": [1]}), + ), + ]; + + for (name, field_type, default) in cases { + let schema_json = serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [{ + "id": 1, + "name": name, + "required": false, + "type": field_type, + "initial-default": default + }] + }); + + assert!( + serde_json::from_value::(schema_json).is_err(), + "non-null unknown default in {name} should be rejected" + ); + } + } } diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 2b9d1d140e..84bb97a328 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -35,7 +35,7 @@ pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataB use super::{ DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, Schema, SchemaId, SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - TableProperties, + TableProperties, Transform, }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; @@ -565,8 +565,26 @@ impl TableMetadata { Ok(self) } - /// If the default partition spec is not present in specs, add it + /// Validate active default-spec sources and add the spec if it is not present. fn try_normalize_partition_spec(&mut self) -> Result<()> { + for field in self.default_spec.fields() { + // Historical specs may reference dropped columns, but active default-spec + // fields need a source for new writes. Void fields do not read their source. + if field.transform != Transform::Void + && self.current_schema().field_by_id(field.source_id).is_none() + { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Default partition spec {} references missing source field {} in current schema {}", + self.default_spec.spec_id(), + field.source_id, + self.current_schema_id + ), + )); + } + } + if self .partition_spec_by_id(self.default_spec.spec_id()) .is_none() @@ -3546,6 +3564,70 @@ mod tests { assert_eq!(FormatVersion::V2, FormatVersion::V2); } + #[test] + fn test_deserialize_default_partition_spec_with_dropped_source() { + for version in [1, 2, 3] { + for transform in ["identity", "truncate[4]", "bucket[16]"] { + let mut metadata = serde_json::json!({ + "format-version": version, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590_i64, + "last-column-id": 2, + "current-schema-id": 1, + "schemas": [ + { + "schema-id": 0, + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": false, "type": "long"}, + {"id": 2, "name": "y", "required": false, "type": "long"} + ] + }, + { + "schema-id": 1, + "type": "struct", + "fields": [ + {"id": 2, "name": "y", "required": false, "type": "long"} + ] + } + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{ + "source-id": 1, "field-id": 1000, "name": "x_partition", + "transform": transform + }]}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "next-row-id": 0 + }); + + // Retaining the source in history does not make it writable using + // the current schema and default spec. + let err = serde_json::from_value::(metadata.clone()).unwrap_err(); + assert!(err.to_string().contains( + "Default partition spec 0 references missing source field 1 in current schema 1" + )); + + // The same spec is allowed once it becomes historical. + metadata["partition-specs"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"spec-id": 1, "fields": []})); + metadata["default-spec-id"] = serde_json::json!(1); + serde_json::from_value::(metadata.clone()).unwrap(); + + // A voided field does not require its dropped source, even in the default spec. + metadata["default-spec-id"] = serde_json::json!(0); + metadata["partition-specs"][0]["fields"][0]["transform"] = + serde_json::json!("void"); + serde_json::from_value::(metadata).unwrap(); + } + } + } + #[test] fn test_default_partition_spec() { let default_spec_id = 1234; diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index d5eccd8328..2890029065 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -368,6 +368,12 @@ impl Datum { /// See [this spec](https://iceberg.apache.org/spec/#binary-single-value-serialization) for reference. pub fn try_from_bytes(bytes: &[u8], data_type: PrimitiveType) -> Result { let literal = match data_type { + PrimitiveType::Unknown => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Cannot create datum for unknown type from bytes", + )); + } PrimitiveType::Boolean => { if bytes.len() == 1 && bytes[0] == 0u8 { PrimitiveLiteral::Boolean(false) diff --git a/crates/iceberg/src/spec/values/literal.rs b/crates/iceberg/src/spec/values/literal.rs index 194b1c3b94..63d6c1aee5 100644 --- a/crates/iceberg/src/spec/values/literal.rs +++ b/crates/iceberg/src/spec/values/literal.rs @@ -547,6 +547,10 @@ impl Literal { decimal_mantissa(&rescaled), )))) } + (PrimitiveType::Unknown, value) if !value.is_null() => Err(Error::new( + ErrorKind::DataInvalid, + "Unknown type only supports null default values", + )), (_, JsonValue::Null) => Ok(None), (i, j) => Err(Error::new( ErrorKind::DataInvalid, @@ -555,20 +559,15 @@ impl Literal { }, Type::Struct(schema) => { if let JsonValue::Object(mut object) = value { - Ok(Some(Literal::Struct(Struct::from_iter( - schema.fields().iter().map(|field| { - object.remove(&field.id.to_string()).and_then(|value| { - Literal::try_from_json(value, &field.field_type) - .and_then(|value| { - value.ok_or(Error::new( - ErrorKind::DataInvalid, - "Key of map cannot be null", - )) - }) - .ok() - }) - }), - )))) + let values = schema + .fields() + .iter() + .map(|field| match object.remove(&field.id.to_string()) { + Some(value) => Literal::try_from_json(value, &field.field_type), + None => Ok(None), + }) + .collect::>>()?; + Ok(Some(Literal::Struct(Struct::from_iter(values)))) } else { Err(Error::new( ErrorKind::DataInvalid, diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index 1d716a48da..97b2f6b325 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -342,6 +342,23 @@ fn json_fixed() { ); } +#[test] +fn json_unknown_only_accepts_null() { + let unknown = Primitive(PrimitiveType::Unknown); + + assert_eq!( + Literal::try_from_json(JsonValue::Null, &unknown).unwrap(), + None + ); + let error = Literal::try_from_json(serde_json::json!(1), &unknown).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!( + error + .message() + .contains("Unknown type only supports null default values") + ); +} + #[test] fn test_should_parse_json_binary_if_hex_uses_uppercase_digits() { let result = Literal::try_from_json( From a1de3a10f0916de2be39b886008b5763329767ac Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:32:32 +0000 Subject: [PATCH 02/63] chore: remove migrated DataFusion integration crates (#3228) Co-authored-by: Kevin Liu --- Cargo.lock | 1773 ++--------------- Cargo.toml | 19 - README.md | 7 - bindings/python/DEPENDENCIES.rust.tsv | 1 - crates/integrations/datafusion/Cargo.toml | 49 - .../datafusion/DEPENDENCIES.rust.tsv | 412 ---- crates/integrations/datafusion/LICENSE | 1 - crates/integrations/datafusion/NOTICE | 1 - crates/integrations/datafusion/README.md | 22 - crates/integrations/datafusion/public-api.txt | 139 -- crates/integrations/datafusion/src/catalog.rs | 93 - crates/integrations/datafusion/src/error.rs | 32 - crates/integrations/datafusion/src/lib.rs | 30 - .../datafusion/src/physical_plan/commit.rs | 713 ------- .../src/physical_plan/expr_to_predicate.rs | 927 --------- .../src/physical_plan/metadata_scan.rs | 98 - .../datafusion/src/physical_plan/mod.rs | 31 - .../datafusion/src/physical_plan/project.rs | 590 ------ .../src/physical_plan/repartition.rs | 853 -------- .../datafusion/src/physical_plan/scan.rs | 252 --- .../datafusion/src/physical_plan/sort.rs | 240 --- .../datafusion/src/physical_plan/write.rs | 653 ------ crates/integrations/datafusion/src/schema.rs | 447 ----- .../datafusion/src/table/metadata_table.rs | 84 - .../integrations/datafusion/src/table/mod.rs | 893 --------- .../src/table/table_provider_factory.rs | 331 --- .../datafusion/src/task_writer.rs | 517 ----- .../table_metadata/TableMetadataV2.json | 121 -- .../tests/integration_datafusion_test.rs | 948 --------- .../tests/test_data/TableMetadataV2Valid.json | 122 -- crates/integrations/playground/Cargo.toml | 53 - .../playground/DEPENDENCIES.rust.tsv | 517 ----- crates/integrations/playground/README.md | 25 - crates/integrations/playground/src/catalog.rs | 204 -- crates/integrations/playground/src/lib.rs | 22 - crates/integrations/playground/src/main.rs | 112 -- crates/sqllogictest/Cargo.toml | 58 - crates/sqllogictest/DEPENDENCIES.rust.tsv | 481 ----- crates/sqllogictest/README.md | 35 - crates/sqllogictest/src/engine/datafusion.rs | 204 -- crates/sqllogictest/src/engine/mod.rs | 143 -- crates/sqllogictest/src/error.rs | 52 - crates/sqllogictest/src/lib.rs | 23 - crates/sqllogictest/src/schedule.rs | 255 --- .../testdata/schedules/df_basic_queries.toml | 23 - .../df_binary_predicate_pushdown.toml | 23 - .../df_boolean_predicate_pushdown.toml | 23 - .../testdata/schedules/df_create_table.toml | 23 - .../testdata/schedules/df_drop_table.toml | 23 - .../df_encrypted_insert_round_trip.toml | 23 - .../schedules/df_insert_and_like.toml | 27 - .../testdata/schedules/df_scan_all_types.toml | 23 - .../testdata/schedules/df_show_tables.toml | 23 - .../df_timestamp_predicate_pushdown.toml | 23 - .../testdata/slts/df_test/basic_queries.slt | 224 --- .../df_test/binary_predicate_pushdown.slt | 67 - .../df_test/boolean_predicate_pushdown.slt | 124 -- .../testdata/slts/df_test/create_table.slt | 90 - .../testdata/slts/df_test/drop_table.slt | 63 - .../df_test/encrypted_insert_round_trip.slt | 31 - .../testdata/slts/df_test/insert_into.slt | 153 -- .../slts/df_test/like_predicate_pushdown.slt | 109 - .../testdata/slts/df_test/scan_all_types.slt | 128 -- .../testdata/slts/df_test/show_tables.slt | 46 - .../df_test/timestamp_predicate_pushdown.slt | 183 -- crates/sqllogictest/tests/sqllogictests.rs | 87 - 66 files changed, 144 insertions(+), 13978 deletions(-) delete mode 100644 crates/integrations/datafusion/Cargo.toml delete mode 100644 crates/integrations/datafusion/DEPENDENCIES.rust.tsv delete mode 120000 crates/integrations/datafusion/LICENSE delete mode 120000 crates/integrations/datafusion/NOTICE delete mode 100644 crates/integrations/datafusion/README.md delete mode 100644 crates/integrations/datafusion/public-api.txt delete mode 100644 crates/integrations/datafusion/src/catalog.rs delete mode 100644 crates/integrations/datafusion/src/error.rs delete mode 100644 crates/integrations/datafusion/src/lib.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/commit.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/metadata_scan.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/mod.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/project.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/repartition.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/scan.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/sort.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/write.rs delete mode 100644 crates/integrations/datafusion/src/schema.rs delete mode 100644 crates/integrations/datafusion/src/table/metadata_table.rs delete mode 100644 crates/integrations/datafusion/src/table/mod.rs delete mode 100644 crates/integrations/datafusion/src/table/table_provider_factory.rs delete mode 100644 crates/integrations/datafusion/src/task_writer.rs delete mode 100644 crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json delete mode 100644 crates/integrations/datafusion/tests/integration_datafusion_test.rs delete mode 100644 crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json delete mode 100644 crates/integrations/playground/Cargo.toml delete mode 100644 crates/integrations/playground/DEPENDENCIES.rust.tsv delete mode 100644 crates/integrations/playground/README.md delete mode 100644 crates/integrations/playground/src/catalog.rs delete mode 100644 crates/integrations/playground/src/lib.rs delete mode 100644 crates/integrations/playground/src/main.rs delete mode 100644 crates/sqllogictest/Cargo.toml delete mode 100644 crates/sqllogictest/DEPENDENCIES.rust.tsv delete mode 100644 crates/sqllogictest/README.md delete mode 100644 crates/sqllogictest/src/engine/datafusion.rs delete mode 100644 crates/sqllogictest/src/engine/mod.rs delete mode 100644 crates/sqllogictest/src/error.rs delete mode 100644 crates/sqllogictest/src/lib.rs delete mode 100644 crates/sqllogictest/src/schedule.rs delete mode 100644 crates/sqllogictest/testdata/schedules/df_basic_queries.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_create_table.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_drop_table.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_insert_and_like.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_scan_all_types.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_show_tables.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml delete mode 100644 crates/sqllogictest/testdata/slts/df_test/basic_queries.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/create_table.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/drop_table.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/insert_into.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/scan_all_types.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/show_tables.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt delete mode 100644 crates/sqllogictest/tests/sqllogictests.rs diff --git a/Cargo.lock b/Cargo.lock index 069acd0d9f..f16f025bb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,7 +174,7 @@ dependencies = [ "serde_json", "snap", "strum", - "strum_macros 0.27.2", + "strum_macros", "thiserror 2.0.18", "uuid", "zstd", @@ -189,15 +189,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ar_archive_writer" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" -dependencies = [ - "object", -] - [[package]] name = "arc-swap" version = "1.9.2" @@ -281,30 +272,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-avro" -version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "bytes", - "bzip2", - "crc", - "flate2", - "indexmap 2.14.0", - "liblzma", - "rand 0.9.5", - "serde", - "serde_json", - "snap", - "strum_macros 0.28.0", - "uuid", - "zstd", -] - [[package]] name = "arrow-buffer" version = "59.3.0" @@ -332,7 +299,6 @@ dependencies = [ "atoi", "base64 0.23.0", "chrono", - "comfy-table", "half", "lexical-core", "num-traits", @@ -379,8 +345,6 @@ dependencies = [ "arrow-schema", "arrow-select", "flatbuffers", - "lz4_flex 0.14.0", - "zstd", ] [[package]] @@ -453,8 +417,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" dependencies = [ "bitflags 2.13.0", - "serde_core", - "serde_json", ] [[package]] @@ -516,18 +478,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-compression" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" -dependencies = [ - "compression-codecs", - "compression-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-lock" version = "3.4.2" @@ -547,7 +497,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -558,7 +508,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -989,7 +939,7 @@ checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1138,15 +1088,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "blake3" version = "1.8.5" @@ -1220,7 +1161,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.119", + "syn", ] [[package]] @@ -1287,7 +1228,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1315,15 +1256,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" -dependencies = [ - "libbz2-rs-sys", -] - [[package]] name = "cbc" version = "0.1.2" @@ -1439,7 +1371,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1448,15 +1380,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "clipboard-win" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" -dependencies = [ - "error-code", -] - [[package]] name = "cmake" version = "0.1.58" @@ -1497,37 +1420,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "comfy-table" -version = "7.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" -dependencies = [ - "unicode-segmentation", - "unicode-width 0.2.2", -] - -[[package]] -name = "compression-codecs" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" -dependencies = [ - "bzip2", - "compression-core", - "flate2", - "liblzma", - "memchr", - "zstd", - "zstd-safe", -] - -[[package]] -name = "compression-core" -version = "0.4.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" - [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1537,18 +1429,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "console" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" -dependencies = [ - "encode_unicode", - "libc", - "unicode-width 0.2.2", - "windows-sys 0.61.2", -] - [[package]] name = "const-oid" version = "0.9.6" @@ -1844,7 +1724,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.119", + "syn", ] [[package]] @@ -1857,7 +1737,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.119", + "syn", ] [[package]] @@ -1868,7 +1748,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1879,7 +1759,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1897,921 +1777,147 @@ dependencies = [ ] [[package]] -name = "datafusion" -version = "55.0.0" +name = "defmt" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" dependencies = [ - "arrow", - "arrow-schema", - "async-trait", - "bzip2", - "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", - "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", - "flate2", - "futures", - "indexmap 2.14.0", - "itertools 0.15.0", - "liblzma", - "log", - "object_store", - "parking_lot", - "parquet", - "sqlparser", - "tempfile", - "tokio", - "url", - "uuid", - "zstd", + "bitflags 1.3.2", + "defmt-macros", ] [[package]] -name = "datafusion-catalog" -version = "55.0.0" +name = "defmt-macros" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.15.0", - "log", - "object_store", - "parking_lot", - "tokio", + "defmt-parser", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "datafusion-catalog-listing" -version = "55.0.0" +name = "defmt-parser" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "futures", - "itertools 0.15.0", - "log", - "object_store", - "percent-encoding", + "thiserror 2.0.18", ] [[package]] -name = "datafusion-cli" -version = "55.0.0" +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8b87ff78e243da48b836ced274799a1799b4a2fd321700e4b8f3426c01aa7e" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "arrow", - "async-trait", - "aws-config", - "aws-credential-types", - "chrono", - "clap", - "datafusion", - "datafusion-common", - "dirs", - "env_logger", - "futures", - "log", - "mimalloc", - "object_store", - "parking_lot", - "parquet", - "regex", - "rustyline", - "tokio", - "url", + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", ] [[package]] -name = "datafusion-common" -version = "55.0.0" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", - "chrono", - "foldhash 0.2.0", - "half", - "hashbrown 0.17.1", - "hex", - "indexmap 2.14.0", - "itertools 0.15.0", - "libc", - "log", - "num-traits", - "object_store", - "parquet", - "recursive", - "sqlparser", - "tokio", - "uuid", - "web-time", + "serde_core", ] [[package]] -name = "datafusion-common-runtime" -version = "55.0.0" +name = "derive_builder" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" dependencies = [ - "futures", - "log", - "tokio", + "derive_builder_macro", ] [[package]] -name = "datafusion-datasource" -version = "55.0.0" +name = "derive_builder_core" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "arrow", - "async-compression", - "async-trait", - "bytes", - "bzip2", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "flate2", - "futures", - "glob", - "itertools 0.15.0", - "liblzma", - "log", - "object_store", - "parking_lot", - "rand 0.9.5", - "tokio", - "tokio-util", - "url", - "zstd", + "darling 0.20.11", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "datafusion-datasource-arrow" -version = "55.0.0" +name = "derive_builder_macro" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ - "arrow", - "arrow-ipc", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.15.0", - "object_store", - "tokio", + "derive_builder_core", + "syn", ] [[package]] -name = "datafusion-datasource-avro" -version = "55.0.0" +name = "diff" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c9587cff8163f9bfcb186e8664ba85cb327031b8ff14330307f961ea2fc196" -dependencies = [ - "arrow", - "arrow-avro", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-adapter", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", -] +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] -name = "datafusion-datasource-csv" -version = "55.0.0" +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "regex", - "tokio", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", ] [[package]] -name = "datafusion-datasource-json" -version = "55.0.0" +name = "digest" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "tokio", - "tokio-stream", + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", ] [[package]] -name = "datafusion-datasource-parquet" -version = "55.0.0" +name = "dirs" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "arrow", - "arrow-schema", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", - "futures", - "itertools 0.15.0", - "log", - "object_store", - "parking_lot", - "parquet", - "tokio", + "dirs-sys", ] [[package]] -name = "datafusion-doc" -version = "55.0.0" +name = "dirs-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] [[package]] -name = "datafusion-execution" -version = "55.0.0" +name = "displaydoc" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d" -dependencies = [ - "arrow", - "arrow-buffer", - "async-trait", - "bytes", - "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", - "futures", - "log", - "object_store", - "parking_lot", - "parquet", - "pin-project-lite", - "rand 0.9.5", - "tempfile", - "tokio", - "tokio-util", - "url", -] - -[[package]] -name = "datafusion-expr" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e" -dependencies = [ - "arrow", - "arrow-schema", - "async-trait", - "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", - "indexmap 2.14.0", - "itertools 0.15.0", - "recursive", - "serde_json", - "sqlparser", -] - -[[package]] -name = "datafusion-expr-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" -dependencies = [ - "arrow", - "datafusion-common", - "indexmap 2.14.0", - "itertools 0.15.0", -] - -[[package]] -name = "datafusion-functions" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde" -dependencies = [ - "arrow", - "arrow-buffer", - "base64 0.23.0", - "blake2", - "blake3", - "chrono", - "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", - "datafusion-physical-expr-common", - "hex", - "itertools 0.15.0", - "log", - "md-5 0.11.0", - "memchr", - "num-traits", - "rand 0.9.5", - "regex", - "sha2 0.11.0", - "uuid", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "half", - "hashbrown 0.17.1", - "log", - "num-traits", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-functions-nested" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", - "hashbrown 0.17.1", - "itertools 0.15.0", - "itoa", - "log", - "memchr", -] - -[[package]] -name = "datafusion-functions-table" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840" -dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "parking_lot", -] - -[[package]] -name = "datafusion-functions-window" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "log", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde" -dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-macros" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" -dependencies = [ - "datafusion-doc", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "datafusion-optimizer" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "indexmap 2.14.0", - "itertools 0.15.0", - "log", - "recursive", - "regex", - "regex-syntax", -] - -[[package]] -name = "datafusion-physical-expr" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", - "half", - "hashbrown 0.17.1", - "indexmap 2.14.0", - "itertools 0.15.0", - "parking_lot", - "petgraph", - "recursive", - "tokio", -] - -[[package]] -name = "datafusion-physical-expr-adapter" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "itertools 0.15.0", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-expr-common", - "hashbrown 0.17.1", - "indexmap 2.14.0", - "itertools 0.15.0", - "parking_lot", - "pin-project", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", - "itertools 0.15.0", - "recursive", -] - -[[package]] -name = "datafusion-physical-plan" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112" -dependencies = [ - "arrow", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "futures", - "half", - "hashbrown 0.17.1", - "indexmap 2.14.0", - "itertools 0.15.0", - "log", - "num-traits", - "parking_lot", - "pin-project-lite", - "serde_json", - "tokio", -] - -[[package]] -name = "datafusion-pruning" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "log", -] - -[[package]] -name = "datafusion-session" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1" -dependencies = [ - "arrow-schema", - "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", - "parking_lot", -] - -[[package]] -name = "datafusion-spark" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "992dd0b954f24cb576cbeee3555c3083b9cf7a5a5f2448ea25f7a437d444163f" -dependencies = [ - "arrow", - "bigdecimal", - "chrono", - "crc32fast", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-functions-nested", - "log", - "num-traits", - "percent-encoding", - "rand 0.9.5", - "serde_json", - "sha1 0.11.0", - "sha2 0.11.0", - "twox-hash", - "url", -] - -[[package]] -name = "datafusion-sql" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9" -dependencies = [ - "arrow", - "bigdecimal", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", - "indexmap 2.14.0", - "log", - "recursive", - "regex", - "sqlparser", - "stacker", -] - -[[package]] -name = "datafusion-sqllogictest" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c921345d249b784a513ad2be71fc3df18fd8f019fe6c82233f085a1871154fd" -dependencies = [ - "arrow", - "async-trait", - "bigdecimal", - "clap", - "datafusion", - "datafusion-spark", - "futures", - "half", - "indicatif", - "itertools 0.15.0", - "log", - "object_store", - "sqllogictest", - "sqlparser", - "tempfile", - "thiserror 2.0.18", - "tokio", -] - -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror 2.0.18", -] - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "serde_core", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.119", -] - -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.4", - "const-oid 0.9.6", - "crypto-common 0.1.7", - "subtle", -] - -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer 0.12.1", - "const-oid 0.10.2", - "crypto-common 0.2.2", - "ctutils", -] - -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.61.2", -] - -[[package]] -name = "displaydoc" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -2868,18 +1974,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "educe" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" -dependencies = [ - "enum-ordinalize", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "either" version = "1.16.0" @@ -2889,61 +1983,6 @@ dependencies = [ "serde", ] -[[package]] -name = "encode_unicode" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" - -[[package]] -name = "endian-type" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" - -[[package]] -name = "enum-ordinalize" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07f808d588c10e464ea6f7d3eaed500049eff30aaac103460f61828c2d65b3eb" -dependencies = [ - "enum-ordinalize-derive", -] - -[[package]] -name = "enum-ordinalize-derive" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "env_filter" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -2971,18 +2010,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "error-code" -version = "3.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" - -[[package]] -name = "escape8259" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" - [[package]] name = "etcetera" version = "0.8.0" @@ -3061,12 +2088,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - [[package]] name = "flatbuffers" version = "25.12.19" @@ -3111,12 +2132,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -3135,15 +2150,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "fs-err" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" -dependencies = [ - "autocfg", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -3217,7 +2223,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3340,7 +2346,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3413,7 +2419,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.1.5", + "foldhash", ] [[package]] @@ -3421,11 +2427,6 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] [[package]] name = "hashlink" @@ -3883,23 +2884,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "iceberg-datafusion" -version = "0.10.1" -dependencies = [ - "anyhow", - "async-trait", - "dashmap", - "datafusion", - "expect-test", - "futures", - "iceberg", - "parquet", - "tempfile", - "tokio", - "uuid", -] - [[package]] name = "iceberg-examples" version = "0.10.1" @@ -3928,28 +2912,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "iceberg-playground" -version = "0.10.1" -dependencies = [ - "anyhow", - "clap", - "datafusion", - "datafusion-cli", - "dirs", - "fs-err", - "home", - "iceberg", - "iceberg-catalog-rest", - "iceberg-datafusion", - "mimalloc", - "stacker", - "tokio", - "toml 0.8.23", - "tracing", - "tracing-subscriber", -] - [[package]] name = "iceberg-property-macro" version = "0.10.1" @@ -3959,32 +2921,10 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.119", + "syn", "trybuild", ] -[[package]] -name = "iceberg-sqllogictest" -version = "0.10.1" -dependencies = [ - "anyhow", - "async-trait", - "datafusion", - "datafusion-sqllogictest", - "enum-ordinalize", - "env_logger", - "iceberg", - "iceberg-datafusion", - "indicatif", - "libtest-mimic", - "log", - "serde", - "sqllogictest", - "tokio", - "toml 0.8.23", - "tracing", -] - [[package]] name = "iceberg-storage-opendal" version = "0.10.1" @@ -4147,19 +3087,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "indicatif" -version = "0.18.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" -dependencies = [ - "console", - "portable-atomic", - "unicode-width 0.2.2", - "unit-prefix", - "web-time", -] - [[package]] name = "inout" version = "0.1.4" @@ -4219,15 +3146,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -4260,7 +3178,7 @@ checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4305,7 +3223,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.119", + "syn", ] [[package]] @@ -4324,7 +3242,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4431,53 +3349,18 @@ dependencies = [ "lexical-util", ] -[[package]] -name = "libbz2-rs-sys" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" - [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "liblzma" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45aec2360b3933207e27908049d8e4df4e476b58180afb1e56b2a4fb72efe4ba" -dependencies = [ - "liblzma-sys", -] - -[[package]] -name = "liblzma-sys" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a046c7f353ba30f810545151e04f63545833803f5b86ee3ddf1517247fe560a5" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "libm" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "libmimalloc-sys" -version = "0.1.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" -dependencies = [ - "cc", -] - [[package]] name = "libredox" version = "0.1.18" @@ -4498,19 +3381,7 @@ checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", - "vcpkg", -] - -[[package]] -name = "libtest-mimic" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" -dependencies = [ - "anstream", - "anstyle", - "clap", - "escape8259", + "vcpkg", ] [[package]] @@ -4680,15 +3551,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "mimalloc" -version = "0.1.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "minijinja" version = "2.24.0" @@ -4743,7 +3605,7 @@ dependencies = [ "cfg-if 1.0.4", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4817,7 +3679,7 @@ checksum = "b40e46c845ac234bcba19db7ab252bc2778cbadd516a466d2f12b1580852d136" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4837,7 +3699,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4852,15 +3714,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" -[[package]] -name = "nibble_vec" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" -dependencies = [ - "smallvec", -] - [[package]] name = "nix" version = "0.29.0" @@ -4874,18 +3727,6 @@ dependencies = [ "memoffset", ] -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.13.0", - "cfg-if 1.0.4", - "cfg_aliases", - "libc", -] - [[package]] name = "ntapi" version = "0.4.3" @@ -5004,7 +3845,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5035,54 +3876,6 @@ dependencies = [ "objc2-core-foundation", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - -[[package]] -name = "object_store" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "chrono", - "form_urlencoded", - "futures-channel", - "futures-core", - "futures-util", - "http 1.5.0", - "http-body-util", - "humantime", - "hyper", - "itertools 0.14.0", - "md-5 0.10.6", - "parking_lot", - "percent-encoding", - "quick-xml 0.39.4", - "rand 0.10.2", - "reqwest 0.12.28", - "ring", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "thiserror 2.0.18", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -5144,7 +3937,7 @@ dependencies = [ "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "serde", "serde_json", @@ -5224,7 +4017,7 @@ dependencies = [ "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.41.0", + "quick-xml", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5268,7 +4061,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -5303,7 +4096,7 @@ dependencies = [ "http 1.5.0", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5323,7 +4116,7 @@ dependencies = [ "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5388,12 +4181,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "owo-colors" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" - [[package]] name = "parking" version = "2.2.1" @@ -5448,7 +4235,6 @@ dependencies = [ "num-bigint 0.5.1", "num-integer", "num-traits", - "object_store", "ring", "seq-macro", "simdutf8", @@ -5499,18 +4285,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "serde", -] - [[package]] name = "phf" version = "0.12.1" @@ -5568,7 +4342,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5727,7 +4501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.119", + "syn", ] [[package]] @@ -5736,7 +4510,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.12+spec-1.1.0", + "toml_edit", ] [[package]] @@ -5748,16 +4522,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "psm" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" -dependencies = [ - "ar_archive_writer", - "cc", -] - [[package]] name = "ptr_meta" version = "0.3.1" @@ -5775,7 +4539,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5829,7 +4593,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5841,7 +4605,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5850,16 +4614,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" -[[package]] -name = "quick-xml" -version = "0.39.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.41.0" @@ -5948,16 +4702,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "radix_trie" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" -dependencies = [ - "endian-type", - "nibble_vec", -] - [[package]] name = "rancor" version = "0.1.2" @@ -6054,26 +4798,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "recursive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" -dependencies = [ - "recursive-proc-macro-impl", - "stacker", -] - -[[package]] -name = "recursive-proc-macro-impl" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" -dependencies = [ - "quote", - "syn 2.0.119", -] - [[package]] name = "redb" version = "3.1.3" @@ -6129,7 +4853,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -6208,7 +4932,7 @@ dependencies = [ "http 1.5.0", "log", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "rust-ini", "serde", @@ -6226,7 +4950,7 @@ dependencies = [ "bytes", "http 1.5.0", "log", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aws-core", "reqsign-core", "serde", @@ -6316,27 +5040,20 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", "http 1.5.0", "http-body 1.0.1", "http-body-util", "hyper", - "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -6442,7 +5159,7 @@ checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -6501,7 +5218,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.119", + "syn", "unicode-ident", ] @@ -6628,27 +5345,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "rustyline" -version = "18.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f6a737db68eb1a8ccff86b584b2fc13eca6a7bb6f78ebc7c529547e3ab9684" -dependencies = [ - "bitflags 2.13.0", - "cfg-if 1.0.4", - "clipboard-win", - "home", - "libc", - "log", - "memchr", - "nix 0.31.3", - "radix_trie", - "unicode-segmentation", - "unicode-width 0.2.2", - "utf8parse", - "windows-sys 0.61.2", -] - [[package]] name = "ryu" version = "1.0.23" @@ -6825,7 +5521,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -6834,7 +5530,6 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ - "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6850,16 +5545,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", -] - -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", + "syn", ] [[package]] @@ -6913,7 +5599,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7154,53 +5840,6 @@ dependencies = [ "der", ] -[[package]] -name = "sqllogictest" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b2262a244037b0b510edbd25a8e6c9fb8d73ee0237fc6cc95a54c16f94a82" -dependencies = [ - "async-trait", - "educe", - "fs-err", - "futures", - "glob", - "humantime", - "itertools 0.13.0", - "libtest-mimic", - "md-5 0.10.6", - "owo-colors", - "rand 0.8.6", - "regex", - "similar", - "subst", - "tempfile", - "thiserror 2.0.18", - "tracing", -] - -[[package]] -name = "sqlparser" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" -dependencies = [ - "log", - "recursive", - "sqlparser_derive", -] - -[[package]] -name = "sqlparser_derive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "sqlx" version = "0.8.6" @@ -7260,7 +5899,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.119", + "syn", ] [[package]] @@ -7281,7 +5920,7 @@ dependencies = [ "sha2 0.10.9", "sqlx-core", "sqlx-sqlite", - "syn 2.0.119", + "syn", "tokio", "url", ] @@ -7394,19 +6033,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "stacker" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" -dependencies = [ - "cc", - "cfg-if 1.0.4", - "libc", - "psm", - "windows-sys 0.61.2", -] - [[package]] name = "static_assertions" version = "1.1.0" @@ -7446,7 +6072,7 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros 0.27.2", + "strum_macros", ] [[package]] @@ -7458,29 +6084,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", -] - -[[package]] -name = "strum_macros" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "subst" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9a86e5144f63c2d18334698269a8bfae6eece345c70b64821ea5b35054ec99" -dependencies = [ - "memchr", - "unicode-width 0.1.14", + "syn", ] [[package]] @@ -7506,17 +6110,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -7534,7 +6127,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7644,7 +6237,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7655,7 +6248,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7756,7 +6349,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7789,7 +6382,6 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util", ] [[package]] @@ -7805,18 +6397,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned 0.6.9", - "toml_datetime 0.6.11", - "toml_edit 0.22.27", -] - [[package]] name = "toml" version = "1.1.3+spec-1.1.0" @@ -7825,20 +6405,11 @@ checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ "indexmap 2.14.0", "serde_core", - "serde_spanned 1.1.1", - "toml_datetime 1.1.1+spec-1.1.0", + "serde_spanned", + "toml_datetime", "toml_parser", "toml_writer", - "winnow 1.0.3", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", + "winnow", ] [[package]] @@ -7850,20 +6421,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.14.0", - "serde", - "serde_spanned 0.6.9", - "toml_datetime 0.6.11", - "toml_write", - "winnow 0.7.15", -] - [[package]] name = "toml_edit" version = "0.25.12+spec-1.1.0" @@ -7871,9 +6428,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap 2.14.0", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime", "toml_parser", - "winnow 1.0.3", + "winnow", ] [[package]] @@ -7882,15 +6439,9 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.3", + "winnow", ] -[[package]] -name = "toml_write" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" - [[package]] name = "toml_writer" version = "1.1.2+spec-1.1.0" @@ -7975,7 +6526,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8048,7 +6599,7 @@ dependencies = [ "serde_json", "target-triple", "termcolor", - "toml 1.1.3+spec-1.1.0", + "toml", ] [[package]] @@ -8056,9 +6607,6 @@ name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -dependencies = [ - "rand 0.9.5", -] [[package]] name = "typed-builder" @@ -8077,7 +6625,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8113,7 +6661,7 @@ checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8149,30 +6697,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "unit-prefix" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" - [[package]] name = "universal-hash" version = "0.5.1" @@ -8263,7 +6787,7 @@ dependencies = [ "metainfo", "motore", "mur3", - "nix 0.29.0", + "nix", "once_cell", "pin-project", "rand 0.9.5", @@ -8411,7 +6935,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn", "wasm-bindgen-shared", ] @@ -8604,7 +7128,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8615,7 +7139,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8820,15 +7344,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "1.0.3" @@ -9038,7 +7553,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -9059,7 +7574,7 @@ checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -9079,7 +7594,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -9119,7 +7634,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d1083a3dc7..661c56b094 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,6 @@ members = [ "crates/integration_tests", "crates/integrations/*", "crates/property-macro", - "crates/sqllogictest", "crates/storage/*", "crates/test_utils", ] @@ -75,16 +74,8 @@ bimap = "0.6" bytes = "1.11" cfg-if = "1" chrono = "0.4.41" -clap = { version = "4.5.48", features = ["derive", "cargo"] } crc32fast = "1" -dashmap = "6" -datafusion = "55.0.0" -datafusion-cli = "55.0.0" -datafusion-sqllogictest = "55.0.0" derive_builder = "0.20" -dirs = "6" -enum-ordinalize = "4.3.0" -env_logger = "0.11.8" expect-test = "1" fastnum = { version = "0.7", default-features = false, features = [ "std", @@ -94,10 +85,8 @@ faststr = "0.2.31" flate2 = "1.1.5" fnv = "1.0.7" form_urlencoded = "1.2.2" -fs-err = "3.1.0" futures = "0.3" hive_metastore = "0.2.0" -home = "0.5.12" http = "1.2" iceberg = { version = "0.10.0", path = "./crates/iceberg" } iceberg-catalog-glue = { version = "0.10.0", path = "./crates/catalog/glue" } @@ -105,16 +94,11 @@ iceberg-catalog-hms = { version = "0.10.0", path = "./crates/catalog/hms" } iceberg-catalog-rest = { version = "0.10.0", path = "./crates/catalog/rest" } iceberg-catalog-s3tables = { version = "0.10.0", path = "./crates/catalog/s3tables" } iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } -iceberg-datafusion = { version = "0.10.0", path = "./crates/integrations/datafusion" } iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } -indicatif = "0.18" itertools = "0.13" -libtest-mimic = "0.8.1" linkedbytes = "0.1.8" -log = "0.4.28" metainfo = "0.7.14" -mimalloc = "0.1.46" minijinja = "2.12.0" mockall = "0.13.1" mockito = "1" @@ -140,9 +124,7 @@ serde_derive = "1.0.219" serde_json = "1.0.142" serde_repr = "0.1.16" serde_with = "3.4" -sqllogictest = "0.29" sqlx = { version = "0.8.1", default-features = false } -stacker = "0.1.20" strum = "0.27.2" syn = "2" tempfile = "3.18" @@ -150,7 +132,6 @@ tokio = { version = "1.47", default-features = false, features = [ "sync", "rt-multi-thread", ] } -toml = "0.8" tracing = "0.1.41" tracing-subscriber = "0.3.20" trybuild = "1" diff --git a/README.md b/README.md index 536562b7b0..fa974dc767 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,6 @@ The Apache Iceberg Rust project is composed of the following components: | [iceberg-catalog-s3tables] | [![iceberg-catalog-s3tables image]][iceberg-catalog-s3tables link] | [![docs release]][iceberg-catalog-s3tables release docs] [![docs dev]][iceberg-catalog-s3tables dev docs] | | [iceberg-catalog-sql] | [![iceberg-catalog-sql image]][iceberg-catalog-sql link] | [![docs release]][iceberg-catalog-sql release docs] [![docs dev]][iceberg-catalog-sql dev docs] | | [iceberg-cache-moka] | [![iceberg-cache-moka image]][iceberg-cache-moka link] | [![docs release]][iceberg-cache-moka release docs] [![docs dev]][iceberg-cache-moka dev docs] | -| [iceberg-datafusion] | [![iceberg-datafusion image]][iceberg-datafusion link] | [![docs release]][iceberg-datafusion release docs] [![docs dev]][iceberg-datafusion dev docs] | | [iceberg-storage-opendal] | [![iceberg-storage-opendal image]][iceberg-storage-opendal link] | [![docs release]][iceberg-storage-opendal release docs] [![docs dev]][iceberg-storage-opendal dev docs] | [docs release]: https://img.shields.io/badge/docs-release-blue @@ -48,12 +47,6 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg release docs]: https://docs.rs/iceberg [iceberg dev docs]: https://rust.iceberg.apache.org/api/iceberg/ -[iceberg-datafusion]: crates/integrations/datafusion/README.md -[iceberg-datafusion image]: https://img.shields.io/crates/v/iceberg-datafusion.svg -[iceberg-datafusion link]: https://crates.io/crates/iceberg-datafusion -[iceberg-datafusion dev docs]: https://rust.iceberg.apache.org/api/iceberg_datafusion/ -[iceberg-datafusion release docs]: https://docs.rs/iceberg-datafusion - [iceberg-catalog-glue]: crates/catalog/glue/README.md [iceberg-catalog-glue image]: https://img.shields.io/crates/v/iceberg-catalog-glue.svg [iceberg-catalog-glue link]: https://crates.io/crates/iceberg-catalog-glue diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv index d38f10dd3f..4f0b8daa21 100644 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ b/bindings/python/DEPENDENCIES.rust.tsv @@ -238,7 +238,6 @@ hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X iceberg@0.10.0 X -iceberg-datafusion@0.10.0 X iceberg-storage-opendal@0.10.0 X iceberg_test_utils@0.10.0 X icu_collections@2.1.1 X diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/integrations/datafusion/Cargo.toml deleted file mode 100644 index 11fb7b1d0b..0000000000 --- a/crates/integrations/datafusion/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -edition = { workspace = true } -homepage = { workspace = true } -name = "iceberg-datafusion" -publish = true -rust-version = { workspace = true } -version = { workspace = true } - -categories = ["database"] -description = "Apache Iceberg DataFusion Integration" -keywords = ["iceberg", "integrations", "datafusion"] -license = { workspace = true } -repository = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -async-trait = { workspace = true } -dashmap = { workspace = true } -datafusion = { workspace = true } -futures = { workspace = true } -iceberg = { workspace = true } -parquet = { workspace = true } -tokio = { workspace = true } -uuid = { workspace = true } - -[dev-dependencies] -expect-test = { workspace = true } -parquet = { workspace = true } -tempfile = { workspace = true } - -[lints] -workspace = true diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv deleted file mode 100644 index 72d6d7fd2c..0000000000 --- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,412 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -ar_archive_writer@0.5.1 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@58.1.0 X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-csv@58.1.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-json@58.1.0 X -arrow-ord@58.3.0 X -arrow-row@58.1.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-compression@0.4.41 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -blake2@0.10.6 X X -blake3@1.8.3 X X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bzip2@0.6.1 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -chrono-tz@0.10.4 X X -cipher@0.4.4 X X -comfy-table@7.2.2 X -compression-codecs@0.4.37 X X -compression-core@0.4.31 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.4.2 X X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -csv@1.4.0 X X -csv-core@0.1.13 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -datafusion@53.1.0 X -datafusion-catalog@53.1.0 X -datafusion-catalog-listing@53.1.0 X -datafusion-common@53.1.0 X -datafusion-common-runtime@53.1.0 X -datafusion-datasource@53.1.0 X -datafusion-datasource-arrow@53.1.0 X -datafusion-datasource-csv@53.1.0 X -datafusion-datasource-json@53.1.0 X -datafusion-datasource-parquet@53.1.0 X -datafusion-doc@53.1.0 X -datafusion-execution@53.1.0 X -datafusion-expr@53.1.0 X -datafusion-expr-common@53.1.0 X -datafusion-functions@53.1.0 X -datafusion-functions-aggregate@53.1.0 X -datafusion-functions-aggregate-common@53.1.0 X -datafusion-functions-nested@53.1.0 X -datafusion-functions-table@53.1.0 X -datafusion-functions-window@53.1.0 X -datafusion-functions-window-common@53.1.0 X -datafusion-macros@53.1.0 X -datafusion-optimizer@53.1.0 X -datafusion-physical-expr@53.1.0 X -datafusion-physical-expr-adapter@53.1.0 X -datafusion-physical-expr-common@53.1.0 X -datafusion-physical-optimizer@53.1.0 X -datafusion-physical-plan@53.1.0 X -datafusion-pruning@53.1.0 X -datafusion-session@53.1.0 X -datafusion-sql@53.1.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -fixedbitset@0.5.7 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -foldhash@0.2.0 X -form_urlencoded@1.2.2 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -humantime@2.3.0 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-datafusion@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.18 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libbz2-rs-sys@0.2.2 X -libc@0.2.183 X X -liblzma@0.4.6 X X -liblzma-sys@0.4.5 X X -libm@0.2.16 X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.10.6 X X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -object@0.37.3 X X -object_store@0.13.2 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -petgraph@0.8.3 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -psm@0.1.30 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -siphasher@1.0.2 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -sqlparser@0.61.0 X -sqlparser_derive@0.5.0 X -stable_deref_trait@1.2.1 X X -stacker@0.1.23 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.27.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.2 X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/datafusion/LICENSE b/crates/integrations/datafusion/LICENSE deleted file mode 120000 index 5853aaea53..0000000000 --- a/crates/integrations/datafusion/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../../LICENSE \ No newline at end of file diff --git a/crates/integrations/datafusion/NOTICE b/crates/integrations/datafusion/NOTICE deleted file mode 120000 index 295f6bdb3a..0000000000 --- a/crates/integrations/datafusion/NOTICE +++ /dev/null @@ -1 +0,0 @@ -../../../NOTICE \ No newline at end of file diff --git a/crates/integrations/datafusion/README.md b/crates/integrations/datafusion/README.md deleted file mode 100644 index 134a8eff4b..0000000000 --- a/crates/integrations/datafusion/README.md +++ /dev/null @@ -1,22 +0,0 @@ - - -# Apache Iceberg DataFusion Integration - -This crate contains the integration of Apache DataFusion and Apache Iceberg. diff --git a/crates/integrations/datafusion/public-api.txt b/crates/integrations/datafusion/public-api.txt deleted file mode 100644 index 5fcb4e85d3..0000000000 --- a/crates/integrations/datafusion/public-api.txt +++ /dev/null @@ -1,139 +0,0 @@ -pub mod iceberg_datafusion -pub mod iceberg_datafusion::metadata_table -pub struct iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -impl iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub async fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan(self) -> datafusion_common::error::Result>> -impl core::clone::Clone for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::clone(&self) -> iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -impl core::fmt::Debug for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _projection: core::option::Option<&'life2 alloc::vec::Vec>, _filters: &'life3 [datafusion_expr::expr::Expr], _limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub mod iceberg_datafusion::physical_plan -pub struct iceberg_datafusion::physical_plan::IcebergTableScan -impl iceberg_datafusion::physical_plan::IcebergTableScan -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::limit(&self) -> core::option::Option -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::predicates(&self) -> core::option::Option<&iceberg::expr::predicate::Predicate> -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::projection(&self) -> core::option::Option<&[alloc::string::String]> -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::snapshot_id(&self) -> core::option::Option -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::table(&self) -> &iceberg::table::Table -impl core::fmt::Debug for iceberg_datafusion::physical_plan::IcebergTableScan -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_physical_plan::display::DisplayAs for iceberg_datafusion::physical_plan::IcebergTableScan -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::fmt_as(&self, _t: datafusion_physical_plan::display::DisplayFormatType, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_physical_plan::execution_plan::ExecutionPlan for iceberg_datafusion::physical_plan::IcebergTableScan -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::apply_expressions(&self, _f: &mut dyn core::ops::function::FnMut(&alloc::sync::Arc) -> datafusion_common::error::Result) -> datafusion_common::error::Result -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::children(&self) -> alloc::vec::Vec<&alloc::sync::Arc<(dyn datafusion_physical_plan::execution_plan::ExecutionPlan + 'static)>> -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::execute(&self, _partition: usize, _context: alloc::sync::Arc) -> datafusion_common::error::Result -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::name(&self) -> &str -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::properties(&self) -> &alloc::sync::Arc -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::with_new_children(self: alloc::sync::Arc, _children: alloc::vec::Vec>) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::physical_plan::convert_filters_to_predicate(filters: &[datafusion_expr::expr::Expr]) -> core::option::Option -pub fn iceberg_datafusion::physical_plan::project_with_partition(input: alloc::sync::Arc, table: &iceberg::table::Table) -> datafusion_common::error::Result> -pub mod iceberg_datafusion::table -pub mod iceberg_datafusion::table::metadata_table -pub struct iceberg_datafusion::table::metadata_table::IcebergMetadataTableProvider -impl iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub async fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan(self) -> datafusion_common::error::Result>> -impl core::clone::Clone for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::clone(&self) -> iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -impl core::fmt::Debug for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _projection: core::option::Option<&'life2 alloc::vec::Vec>, _filters: &'life3 [datafusion_expr::expr::Expr], _limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub mod iceberg_datafusion::table::table_provider_factory -pub struct iceberg_datafusion::table::table_provider_factory::IcebergTableProviderFactory -impl iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new() -> Self -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new_with_storage_factory(storage_factory: alloc::sync::Arc) -> Self -impl core::default::Default for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub struct iceberg_datafusion::table::IcebergStaticTableProvider -impl iceberg_datafusion::IcebergStaticTableProvider -pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table(table: iceberg::table::Table) -> iceberg::error::Result -pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table_snapshot(table: iceberg::table::Table, snapshot_id: i64) -> iceberg::error::Result -impl core::clone::Clone for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::clone(&self) -> iceberg_datafusion::IcebergStaticTableProvider -impl core::fmt::Debug for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub fn iceberg_datafusion::IcebergStaticTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::IcebergStaticTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::IcebergStaticTableProvider::supports_filters_pushdown(&self, filters: &[&datafusion_expr::expr::Expr]) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::IcebergStaticTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub struct iceberg_datafusion::table::IcebergTableProvider -impl core::clone::Clone for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::clone(&self) -> iceberg_datafusion::IcebergTableProvider -impl core::fmt::Debug for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, state: &'life1 dyn datafusion_session::session::Session, input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub fn iceberg_datafusion::IcebergTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::IcebergTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::IcebergTableProvider::supports_filters_pushdown(&self, filters: &[&datafusion_expr::expr::Expr]) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::IcebergTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub mod iceberg_datafusion::table_provider_factory -pub struct iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -impl iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new() -> Self -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new_with_storage_factory(storage_factory: alloc::sync::Arc) -> Self -impl core::default::Default for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub struct iceberg_datafusion::IcebergCatalogProvider -impl iceberg_datafusion::IcebergCatalogProvider -pub async fn iceberg_datafusion::IcebergCatalogProvider::try_new(client: alloc::sync::Arc) -> iceberg::error::Result -impl core::fmt::Debug for iceberg_datafusion::IcebergCatalogProvider -pub fn iceberg_datafusion::IcebergCatalogProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::catalog::CatalogProvider for iceberg_datafusion::IcebergCatalogProvider -pub fn iceberg_datafusion::IcebergCatalogProvider::schema(&self, name: &str) -> core::option::Option> -pub fn iceberg_datafusion::IcebergCatalogProvider::schema_names(&self) -> alloc::vec::Vec -pub struct iceberg_datafusion::IcebergStaticTableProvider -impl iceberg_datafusion::IcebergStaticTableProvider -pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table(table: iceberg::table::Table) -> iceberg::error::Result -pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table_snapshot(table: iceberg::table::Table, snapshot_id: i64) -> iceberg::error::Result -impl core::clone::Clone for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::clone(&self) -> iceberg_datafusion::IcebergStaticTableProvider -impl core::fmt::Debug for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub fn iceberg_datafusion::IcebergStaticTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::IcebergStaticTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::IcebergStaticTableProvider::supports_filters_pushdown(&self, filters: &[&datafusion_expr::expr::Expr]) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::IcebergStaticTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub struct iceberg_datafusion::IcebergTableProvider -impl core::clone::Clone for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::clone(&self) -> iceberg_datafusion::IcebergTableProvider -impl core::fmt::Debug for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, state: &'life1 dyn datafusion_session::session::Session, input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub fn iceberg_datafusion::IcebergTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::IcebergTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::IcebergTableProvider::supports_filters_pushdown(&self, filters: &[&datafusion_expr::expr::Expr]) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::IcebergTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub struct iceberg_datafusion::IcebergTableProviderFactory -impl iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new() -> Self -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new_with_storage_factory(storage_factory: alloc::sync::Arc) -> Self -impl core::default::Default for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_datafusion::from_datafusion_error(error: datafusion_common::error::DataFusionError) -> iceberg::error::Error -pub fn iceberg_datafusion::to_datafusion_error(error: iceberg::error::Error) -> datafusion_common::error::DataFusionError diff --git a/crates/integrations/datafusion/src/catalog.rs b/crates/integrations/datafusion/src/catalog.rs deleted file mode 100644 index 2c6e1ff002..0000000000 --- a/crates/integrations/datafusion/src/catalog.rs +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::sync::Arc; - -use datafusion::catalog::{CatalogProvider, SchemaProvider}; -use futures::future::try_join_all; -use iceberg::{Catalog, NamespaceIdent, Result}; - -use crate::schema::IcebergSchemaProvider; - -/// Provides an interface to manage and access multiple schemas -/// within an Iceberg [`Catalog`]. -/// -/// Acts as a centralized catalog provider that aggregates -/// multiple [`SchemaProvider`], each associated with distinct namespaces. -#[derive(Debug)] -pub struct IcebergCatalogProvider { - /// A `HashMap` where keys are namespace names - /// and values are dynamic references to objects implementing the - /// [`SchemaProvider`] trait. - schemas: HashMap>, -} - -impl IcebergCatalogProvider { - /// Asynchronously tries to construct a new [`IcebergCatalogProvider`] - /// using the given client to fetch and initialize schema providers for - /// each namespace in the Iceberg [`Catalog`]. - /// - /// This method retrieves the list of namespace names - /// attempts to create a schema provider for each namespace, and - /// collects these providers into a `HashMap`. - pub async fn try_new(client: Arc) -> Result { - // TODO: - // Schemas and providers should be cached and evicted based on time - // As of right now; schemas might become stale. - let schema_names: Vec<_> = client - .list_namespaces(None) - .await? - .iter() - .flat_map(|ns| ns.as_ref().clone()) - .collect(); - - let providers = try_join_all( - schema_names - .iter() - .map(|name| { - IcebergSchemaProvider::try_new( - client.clone(), - NamespaceIdent::new(name.clone()), - ) - }) - .collect::>(), - ) - .await?; - - let schemas: HashMap> = schema_names - .into_iter() - .zip(providers) - .map(|(name, provider)| { - let provider = Arc::new(provider) as Arc; - (name, provider) - }) - .collect(); - - Ok(IcebergCatalogProvider { schemas }) - } -} - -impl CatalogProvider for IcebergCatalogProvider { - fn schema_names(&self) -> Vec { - self.schemas.keys().cloned().collect() - } - - fn schema(&self, name: &str) -> Option> { - self.schemas.get(name).cloned() - } -} diff --git a/crates/integrations/datafusion/src/error.rs b/crates/integrations/datafusion/src/error.rs deleted file mode 100644 index 5bc47013dd..0000000000 --- a/crates/integrations/datafusion/src/error.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use anyhow::anyhow; -use iceberg::{Error, ErrorKind}; - -/// Converts a datafusion error into an iceberg error. -pub fn from_datafusion_error(error: datafusion::error::DataFusionError) -> Error { - Error::new( - ErrorKind::Unexpected, - "Operation failed for hitting datafusion error".to_string(), - ) - .with_source(anyhow!("datafusion error: {error:?}")) -} -/// Converts an iceberg error into a datafusion error. -pub fn to_datafusion_error(error: Error) -> datafusion::error::DataFusionError { - datafusion::error::DataFusionError::External(error.into()) -} diff --git a/crates/integrations/datafusion/src/lib.rs b/crates/integrations/datafusion/src/lib.rs deleted file mode 100644 index 4b0ea8606d..0000000000 --- a/crates/integrations/datafusion/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod catalog; -pub use catalog::*; - -mod error; -pub use error::*; - -pub mod physical_plan; -mod schema; -pub mod table; -pub use table::table_provider_factory::IcebergTableProviderFactory; -pub use table::*; - -pub(crate) mod task_writer; diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs deleted file mode 100644 index bb507217e2..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ /dev/null @@ -1,713 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fmt::{Debug, Formatter}; -use std::sync::Arc; - -use datafusion::arrow::array::{Array, ArrayRef, RecordBatch, StringArray, UInt64Array}; -use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, -}; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::common::{DataFusionError, Result as DFResult}; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use futures::StreamExt; -use iceberg::Catalog; -use iceberg::spec::{DataFile, deserialize_data_file_from_json}; -use iceberg::table::Table; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; - -use crate::physical_plan::DATA_FILES_COL_NAME; -use crate::to_datafusion_error; - -/// IcebergCommitExec is responsible for collecting the files written and use -/// [`Transaction::fast_append`] to commit the data files written. -#[derive(Debug)] -pub(crate) struct IcebergCommitExec { - table: Table, - catalog: Arc, - input: Arc, - schema: ArrowSchemaRef, - count_schema: ArrowSchemaRef, - plan_properties: Arc, -} - -impl IcebergCommitExec { - pub fn new( - table: Table, - catalog: Arc, - input: Arc, - schema: ArrowSchemaRef, - ) -> Self { - let count_schema = Self::make_count_schema(); - - let plan_properties = Self::compute_properties(Arc::clone(&count_schema)); - - Self { - table, - catalog, - input, - schema, - count_schema, - plan_properties, - } - } - - // Compute the plan properties for this execution plan - fn compute_properties(schema: ArrowSchemaRef) -> Arc { - Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - )) - } - - // Create a record batch with just the count of rows written - fn make_count_batch(count: u64) -> DFResult { - let count_array = Arc::new(UInt64Array::from(vec![count])) as ArrayRef; - - RecordBatch::try_from_iter_with_nullable(vec![("count", count_array, false)]).map_err(|e| { - DataFusionError::ArrowError( - Box::new(e), - Some("Failed to make count batch!".to_string()), - ) - }) - } - - fn make_count_schema() -> ArrowSchemaRef { - // Define a schema. - Arc::new(ArrowSchema::new(vec![Field::new( - "count", - DataType::UInt64, - false, - )])) - } -} - -impl DisplayAs for IcebergCommitExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default => { - write!(f, "IcebergCommitExec: table={}", self.table.identifier()) - } - DisplayFormatType::Verbose => { - write!( - f, - "IcebergCommitExec: table={}, schema={:?}", - self.table.identifier(), - self.schema - ) - } - DisplayFormatType::TreeRender => { - write!(f, "IcebergCommitExec: table={}", self.table.identifier()) - } - } - } -} - -impl ExecutionPlan for IcebergCommitExec { - fn name(&self) -> &str { - "IcebergCommitExec" - } - - fn properties(&self) -> &Arc { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.input] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn required_input_distribution(&self) -> Vec { - vec![datafusion::physical_plan::Distribution::SinglePartition; self.children().len()] - } - - fn benefits_from_input_partitioning(&self) -> Vec { - vec![false] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> DFResult> { - if children.len() != 1 { - return Err(DataFusionError::Internal(format!( - "IcebergCommitExec expects exactly one child, but provided {}", - children.len() - ))); - } - - Ok(Arc::new(IcebergCommitExec::new( - self.table.clone(), - self.catalog.clone(), - children[0].clone(), - self.schema.clone(), - ))) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> DFResult { - // IcebergCommitExec only has one partition (partition 0) - if partition != 0 { - return Err(DataFusionError::Internal(format!( - "IcebergCommitExec only has one partition, but got partition {partition}" - ))); - } - - let table = self.table.clone(); - let input_plan = self.input.clone(); - - // todo revisit this - let spec_id = self.table.metadata().default_partition_spec_id(); - let partition_type = self.table.metadata().default_partition_type().clone(); - let current_schema = self.table.metadata().current_schema().clone(); - - let catalog = Arc::clone(&self.catalog); - - // Process the input streams from all partitions and commit the data files - let stream = futures::stream::once(async move { - let mut data_files: Vec = Vec::new(); - let mut total_record_count: u64 = 0; - - // Execute and collect results from the input coalesced plan - let mut batch_stream = input_plan.execute(0, context)?; - - while let Some(batch_result) = batch_stream.next().await { - let batch = batch_result?; - - let files_array = batch - .column_by_name(DATA_FILES_COL_NAME) - .ok_or_else(|| { - DataFusionError::Internal( - "Expected 'data_files' column in input batch".to_string(), - ) - })? - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "Expected 'data_files' column to be StringArray".to_string(), - ) - })?; - - // Deserialize all data files from the StringArray - let batch_files: Vec = files_array - .into_iter() - .flatten() - .map(|f| -> DFResult { - // Parse JSON to DataFileSerde and convert to DataFile - deserialize_data_file_from_json( - f, - spec_id, - &partition_type, - ¤t_schema, - ) - .map_err(to_datafusion_error) - }) - .collect::>()?; - - // add record_counts from the current batch to total record count - total_record_count += batch_files.iter().map(|f| f.record_count()).sum::(); - - // Add all deserialized files to our collection - data_files.extend(batch_files); - } - - // If no data files were collected, return a single row with count = 0 - if data_files.is_empty() { - return Self::make_count_batch(0); - } - - // Create a transaction and commit the data files - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(data_files); - - // Apply the action and commit the transaction - let _updated_table = action - .apply(tx) - .map_err(to_datafusion_error)? - .commit(catalog.as_ref()) - .await - .map_err(to_datafusion_error)?; - - Self::make_count_batch(total_record_count) - }) - .boxed(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.count_schema), - stream, - ))) - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::fmt; - use std::sync::Arc; - - use datafusion::arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray, UInt64Array}; - use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; - use datafusion::datasource::MemTable; - use datafusion::execution::context::TaskContext; - use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; - use datafusion::physical_plan::common::collect; - use datafusion::physical_plan::execution_plan::Boundedness; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; - use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; - use datafusion::prelude::*; - use futures::StreamExt; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, NestedField, PrimitiveType, Schema, - Struct, Type, - }; - use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; - - use super::*; - use crate::physical_plan::DATA_FILES_COL_NAME; - use crate::table::IcebergTableProvider; - - // A mock execution plan that returns record batches with serialized data files - #[derive(Debug)] - struct MockWriteExec { - schema: Arc, - data_files_json: Vec, - plan_properties: Arc, - } - - impl MockWriteExec { - fn new(data_files_json: Vec) -> Self { - let schema = Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])); - - let plan_properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema.clone()), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - )); - - Self { - schema, - data_files_json, - plan_properties, - } - } - } - - impl ExecutionPlan for MockWriteExec { - fn name(&self) -> &str { - "MockWriteExec" - } - - fn schema(&self) -> Arc { - self.schema.clone() - } - - fn properties(&self) -> &Arc { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> datafusion::common::Result> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> datafusion::common::Result { - // Create a record batch with the serialized data files - let array = Arc::new(StringArray::from(self.data_files_json.clone())) as ArrayRef; - let batch = RecordBatch::try_new(self.schema.clone(), vec![array])?; - - // Create a stream that returns this batch - let stream = futures::stream::once(async move { Ok(batch) }).boxed(); - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - stream, - ))) - } - } - - // Implement DisplayAs for MockDataFilesExec - impl DisplayAs for MockWriteExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> fmt::Result { - match t { - DisplayFormatType::Default - | DisplayFormatType::Verbose - | DisplayFormatType::TreeRender => { - write!(f, "MockDataFilesExec: files={}", self.data_files_json.len()) - } - } - } - } - - #[tokio::test] - async fn test_iceberg_commit_exec() -> Result<(), Box> { - // Create a memory catalog with in-memory file IO - let catalog = Arc::new( - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://root".to_string(), - )]), - ) - .await - .unwrap(), - ); - - // Create a namespace - let namespace = NamespaceIdent::new("test_namespace".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - // Create a schema for the table - let schema = Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - // Create a table - let table_creation = TableCreation::builder() - .name("test_table".to_string()) - .schema(schema) - .location("memory://root/test_table".to_string()) - .properties(HashMap::new()) - .build(); - - let table = catalog.create_table(&namespace, table_creation).await?; - - // Create data files - let data_file1 = DataFileBuilder::default() - .content(DataContentType::Data) - .file_path("path/to/file1.parquet".to_string()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(1024) - .record_count(100) - .partition_spec_id(table.metadata().default_partition_spec_id()) - .partition(Struct::empty()) - .build()?; - - let data_file2 = DataFileBuilder::default() - .content(DataContentType::Data) - .file_path("path/to/file2.parquet".to_string()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(2048) - .record_count(200) - .partition_spec_id(table.metadata().default_partition_spec_id()) - .partition(Struct::empty()) - .build()?; - - // Serialize data files to JSON - let partition_type = table.metadata().default_partition_type().clone(); - let data_file1_json = iceberg::spec::serialize_data_file_to_json( - data_file1.clone(), - &partition_type, - table.metadata().format_version(), - )?; - - let data_file2_json = iceberg::spec::serialize_data_file_to_json( - data_file2.clone(), - &partition_type, - table.metadata().format_version(), - )?; - - // Create a mock execution plan that returns the serialized data files - let input_exec = Arc::new(MockWriteExec::new(vec![data_file1_json, data_file2_json])); - - // Create the IcebergCommitExec - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])); - - let commit_exec = - IcebergCommitExec::new(table.clone(), catalog.clone(), input_exec, arrow_schema); - - // Verify Execution Plan schema matches the count schema - assert_eq!(commit_exec.schema(), IcebergCommitExec::make_count_schema()); - - // Execute the commit exec - let task_ctx = Arc::new(TaskContext::default()); - let stream = commit_exec.execute(0, task_ctx)?; - let batches = collect(stream).await?; - - // Verify the results - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert_eq!(batch.num_columns(), 1); - assert_eq!(batch.num_rows(), 1); - - // The output should be a record batch with a single column "count" and a single row - // with the total record count (100 + 200 = 300) - let count_array = batch.column(0); - assert_eq!(count_array.len(), 1); - assert_eq!(count_array.data_type(), &DataType::UInt64); - - // Verify that the count is correct - let count = count_array.as_any().downcast_ref::().unwrap(); - assert_eq!(count.value(0), 300); - - // Verify that the table has been updated with the new files - let updated_table = catalog - .load_table(&TableIdent::from_strs(["test_namespace", "test_table"]).unwrap()) - .await?; - let current_snapshot = updated_table.metadata().current_snapshot().unwrap(); - - // Load the manifest list to verify the data files were added - let manifest_list = updated_table - .manifest_list_reader(current_snapshot) - .load() - .await?; - - // There should be at least one manifest - assert!(!manifest_list.entries().is_empty()); - - // Load the first manifest and verify it contains our data files - let manifest = updated_table - .manifest_reader() - .read(&manifest_list.entries()[0]) - .await?; - - // Verify that the manifest contains our data files - let manifest_files: Vec = manifest - .entries() - .iter() - .map(|entry| entry.data_file().file_path().to_string()) - .collect(); - - assert!(manifest_files.contains(&"path/to/file1.parquet".to_string())); - assert!(manifest_files.contains(&"path/to/file2.parquet".to_string())); - - Ok(()) - } - - #[tokio::test] - async fn test_iceberg_commit_exec_empty_insert() -> Result<(), Box> { - let catalog = Arc::new( - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://root".to_string(), - )]), - ) - .await - .unwrap(), - ); - - let namespace = NamespaceIdent::new("test_empty_insert".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - let schema = Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - ]) - .build()?; - - let table_creation = TableCreation::builder() - .name("empty_insert_table".to_string()) - .schema(schema) - .location("memory://root/empty_insert_table".to_string()) - .properties(HashMap::new()) - .build(); - - let table = catalog.create_table(&namespace, table_creation).await?; - let snapshot_count_before = table.metadata().snapshots().len(); - - // Mock write plan produces no data files - let input_exec = Arc::new(MockWriteExec::new(vec![])); - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])); - let commit_exec = - IcebergCommitExec::new(table.clone(), catalog.clone(), input_exec, arrow_schema); - - let task_ctx = Arc::new(TaskContext::default()); - let stream = commit_exec.execute(0, task_ctx)?; - let batches = collect(stream).await?; - - // Must return exactly one batch with one row and one column - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert_eq!(batch.num_rows(), 1); - assert_eq!(batch.num_columns(), 1); - - // The count column must be UInt64 with value 0 - let count_array = batch.column(0); - assert_eq!(count_array.data_type(), &DataType::UInt64); - let count = count_array.as_any().downcast_ref::().unwrap(); - assert_eq!(count.value(0), 0); - - // No new snapshot should be created for an empty insert - let updated_table = catalog - .load_table( - &TableIdent::from_strs(["test_empty_insert", "empty_insert_table"]).unwrap(), - ) - .await?; - let snapshot_count_after = updated_table.metadata().snapshots().len(); - assert_eq!( - snapshot_count_after, snapshot_count_before, - "Empty insert must not create a new snapshot" - ); - assert!(updated_table.metadata().current_snapshot().is_none()); - - Ok(()) - } - - #[tokio::test] - async fn test_datafusion_execution_partitioned_source() -> Result<(), Box> - { - let catalog = Arc::new( - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://root".to_string(), - )]), - ) - .await?, - ); - - let namespace = NamespaceIdent::new("test_namespace".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - let schema = Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - let table_name = "test_table"; - let table_creation = TableCreation::builder() - .name(table_name.to_string()) - .schema(schema) - .location("memory://root/test_table".to_string()) - .properties(HashMap::new()) - .build(); - let _ = catalog.create_table(&namespace, table_creation).await?; - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - - let batches: Vec = (1..4) - .map(|idx| { - RecordBatch::try_new(arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![idx])) as ArrayRef, - Arc::new(StringArray::from(vec![format!("Name{idx}")])) as ArrayRef, - ]) - }) - .collect::>()?; - - // Create DataFusion context with specific partition configuration - let mut config = SessionConfig::new(); - config = config.set_usize("datafusion.execution.target_partitions", 8); - let ctx = SessionContext::new_with_config(config); - - // Create multiple partitions - each batch becomes a separate partition - let partitions: Vec> = - batches.into_iter().map(|batch| vec![batch]).collect(); - let source_table = Arc::new(MemTable::try_new(Arc::clone(&arrow_schema), partitions)?); - ctx.register_table("source_table", source_table)?; - - let iceberg_table_provider = IcebergTableProvider::try_new( - catalog.clone(), - namespace.clone(), - table_name.to_string(), - ) - .await?; - ctx.register_table("iceberg_table", Arc::new(iceberg_table_provider))?; - - let insert_plan = ctx - .sql("INSERT INTO iceberg_table SELECT * FROM source_table") - .await?; - - let physical_plan = insert_plan.create_physical_plan().await?; - - let actual_plan = format!( - "{}", - datafusion::physical_plan::displayable(physical_plan.as_ref()).indent(false) - ); - - println!("Physical plan:\n{actual_plan}"); - - let expected_plan = "\ -IcebergCommitExec: table=test_namespace.test_table - CoalescePartitionsExec - IcebergWriteExec: table=test_namespace.test_table - DataSourceExec: partitions=3, partition_sizes=[1, 1, 1]"; - - assert_eq!( - actual_plan.trim(), - expected_plan.trim(), - "Physical plan does not match expected\n\nExpected:\n{}\n\nActual:\n{}", - expected_plan.trim(), - actual_plan.trim() - ); - - Ok(()) - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs b/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs deleted file mode 100644 index fb5440a98a..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs +++ /dev/null @@ -1,927 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::vec; - -use datafusion::arrow::datatypes::DataType; -use datafusion::logical_expr::expr::ScalarFunction; -use datafusion::logical_expr::{BinaryExpr, Expr, Like, Operator}; -use datafusion::scalar::ScalarValue; -use iceberg::expr::{BinaryExpression, Predicate, PredicateOperator, Reference, UnaryExpression}; -use iceberg::spec::{Datum, PrimitiveLiteral}; - -// A datafusion expression could be an Iceberg predicate, column, or literal. -enum TransformedResult { - Predicate(Predicate), - Column(Reference), - Literal(Datum), - NotTransformed, -} - -enum OpTransformedResult { - Operator(PredicateOperator), - And, - Or, - NotTransformed, -} - -/// Converts DataFusion filters ([`Expr`]) to an iceberg [`Predicate`]. -/// If none of the filters could be converted, return `None` which adds no predicates to the scan operation. -/// If the conversion was successful, return the converted predicates combined with an AND operator. -pub fn convert_filters_to_predicate(filters: &[Expr]) -> Option { - filters - .iter() - .filter_map(convert_filter_to_predicate) - .reduce(Predicate::and) -} - -fn convert_filter_to_predicate(expr: &Expr) -> Option { - match to_iceberg_predicate(expr) { - TransformedResult::Predicate(predicate) => Some(predicate), - TransformedResult::Column(column) => { - // A bare column in a filter context represents a boolean column check - // Convert it to: column = true - Some(Predicate::Binary(BinaryExpression::new( - PredicateOperator::Eq, - column, - Datum::bool(true), - ))) - } - TransformedResult::Literal(_) => { - // Literal values in filter context cannot be pushed down - None - } - _ => None, - } -} - -fn to_iceberg_predicate(expr: &Expr) -> TransformedResult { - match expr { - Expr::BinaryExpr(binary) => { - let left = to_iceberg_predicate(&binary.left); - let right = to_iceberg_predicate(&binary.right); - let op = to_iceberg_operation(binary.op); - match op { - OpTransformedResult::Operator(op) => to_iceberg_binary_predicate(left, right, op), - OpTransformedResult::And => to_iceberg_and_predicate(left, right), - OpTransformedResult::Or => to_iceberg_or_predicate(left, right), - OpTransformedResult::NotTransformed => TransformedResult::NotTransformed, - } - } - Expr::Not(exp) => { - let expr = to_iceberg_predicate(exp); - match expr { - TransformedResult::Predicate(p) => TransformedResult::Predicate(!p), - TransformedResult::Column(column) => { - // NOT of a bare boolean column: NOT col => col = false - TransformedResult::Predicate(Predicate::Binary(BinaryExpression::new( - PredicateOperator::Eq, - column, - Datum::bool(false), - ))) - } - _ => TransformedResult::NotTransformed, - } - } - Expr::Column(column) => TransformedResult::Column(Reference::new(column.name())), - Expr::Literal(literal, _) => match scalar_value_to_datum(literal) { - Some(data) => TransformedResult::Literal(data), - None => TransformedResult::NotTransformed, - }, - Expr::InList(inlist) => { - let mut datums = vec![]; - for expr in &inlist.list { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Literal(l) => datums.push(l), - _ => return TransformedResult::NotTransformed, - } - } - - let expr = to_iceberg_predicate(&inlist.expr); - match expr { - TransformedResult::Column(r) => match inlist.negated { - false => TransformedResult::Predicate(r.is_in(datums)), - true => TransformedResult::Predicate(r.is_not_in(datums)), - }, - _ => TransformedResult::NotTransformed, - } - } - Expr::IsNull(expr) => { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Column(r) => TransformedResult::Predicate(Predicate::Unary( - UnaryExpression::new(PredicateOperator::IsNull, r), - )), - _ => TransformedResult::NotTransformed, - } - } - Expr::IsNotNull(expr) => { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Column(r) => TransformedResult::Predicate(Predicate::Unary( - UnaryExpression::new(PredicateOperator::NotNull, r), - )), - _ => TransformedResult::NotTransformed, - } - } - Expr::Cast(c) => { - if *c.field.data_type() == DataType::Date32 || *c.field.data_type() == DataType::Date64 - { - // Casts to date truncate the expression, we cannot simply extract it as it - // can create erroneous predicates. - return TransformedResult::NotTransformed; - } - to_iceberg_predicate(&c.expr) - } - Expr::Like(Like { - negated, - expr, - pattern, - escape_char, - case_insensitive, - }) => { - // Only support simple prefix patterns (e.g., 'prefix%') - // Note: Iceberg's StartsWith operator is case-sensitive, so we cannot - // push down case-insensitive LIKE (ILIKE) patterns - // Escape characters are also not supported for pushdown - if escape_char.is_some() || *case_insensitive { - return TransformedResult::NotTransformed; - } - - // Extract the pattern string - let pattern_str = match to_iceberg_predicate(pattern) { - TransformedResult::Literal(d) => match d.literal() { - PrimitiveLiteral::String(s) => s.clone(), - _ => return TransformedResult::NotTransformed, - }, - _ => return TransformedResult::NotTransformed, - }; - - // Check if it's a simple prefix pattern (ends with % and no other wildcards) - if pattern_str.ends_with('%') - && !pattern_str[..pattern_str.len() - 1].contains(['%', '_']) - { - // Extract the prefix (remove trailing %) - let prefix = pattern_str[..pattern_str.len() - 1].to_string(); - - // Get the column reference - let column = match to_iceberg_predicate(expr) { - TransformedResult::Column(r) => r, - _ => return TransformedResult::NotTransformed, - }; - - // Create the appropriate predicate - let predicate = if *negated { - column.not_starts_with(Datum::string(prefix)) - } else { - column.starts_with(Datum::string(prefix)) - }; - - TransformedResult::Predicate(predicate) - } else { - // Complex LIKE patterns cannot be pushed down - TransformedResult::NotTransformed - } - } - Expr::ScalarFunction(ScalarFunction { func, args }) => { - scalar_function_to_iceberg_predicate(func.name(), args) - } - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_operation(op: Operator) -> OpTransformedResult { - match op { - Operator::Eq => OpTransformedResult::Operator(PredicateOperator::Eq), - Operator::NotEq => OpTransformedResult::Operator(PredicateOperator::NotEq), - Operator::Lt => OpTransformedResult::Operator(PredicateOperator::LessThan), - Operator::LtEq => OpTransformedResult::Operator(PredicateOperator::LessThanOrEq), - Operator::Gt => OpTransformedResult::Operator(PredicateOperator::GreaterThan), - Operator::GtEq => OpTransformedResult::Operator(PredicateOperator::GreaterThanOrEq), - // AND OR - Operator::And => OpTransformedResult::And, - Operator::Or => OpTransformedResult::Or, - // Others not supported - _ => OpTransformedResult::NotTransformed, - } -} - -/// Translates a DataFusion scalar function into an Iceberg predicate. -/// Unlike dedicated Expr variants (e.g. `Expr::IsNull`), scalar functions are -/// identified by name at runtime, so we need to handle them here. -fn scalar_function_to_iceberg_predicate(func_name: &str, args: &[Expr]) -> TransformedResult { - match func_name { - "isnan" if args.len() == 1 => match resolve_nan_preserving_reference(&args[0]) { - Some(r) => TransformedResult::Predicate(r.is_nan()), - None => TransformedResult::NotTransformed, - }, - _ => TransformedResult::NotTransformed, - } -} - -/// Attempts to resolve a numeric expression argument down to a single column -/// [`Reference`] such that `isnan(arg)` is logically equivalent to -/// `isnan(reference)`. -/// -/// Filter pushdown is reported as `Inexact` (see -/// [`IcebergTableProvider::supports_filters_pushdown`]), so DataFusion -/// re-applies the original predicate after scanning. We therefore only need the -/// pushed-down predicate to be implied by the original filter (it may match -/// extra rows, but must never drop a matching one). Every transformation handled -/// here preserves NaN-ness *exactly* — the result is NaN if and only if the -/// wrapped column is NaN — so both `isnan(arg)` and `NOT isnan(arg)` are sound: -/// -/// * negation: `-x` is NaN iff `x` is NaN -/// * `abs(x)`: `abs(x)` is NaN iff `x` is NaN -/// * casts between numeric types preserve NaN -/// * `x + c`, `c + x`, `x - c`, `c - x` for a finite literal `c` -/// * `x * c`, `c * x`, `x / c` for a finite, non-zero literal `c` -/// -/// Multiplication/division by zero and `c / x` are intentionally rejected: e.g. -/// `x * 0` is NaN when `x` is `±inf`, so it does not imply `x` is NaN. -/// -/// [`IcebergTableProvider::supports_filters_pushdown`]: crate::table::IcebergTableProvider -fn resolve_nan_preserving_reference(expr: &Expr) -> Option { - match expr { - Expr::Column(column) => Some(Reference::new(column.name())), - Expr::Negative(inner) => resolve_nan_preserving_reference(inner), - Expr::Cast(cast) => { - // Casts to date truncate the value and are not numeric, so they - // cannot be treated as NaN-preserving. - if *cast.field.data_type() == DataType::Date32 - || *cast.field.data_type() == DataType::Date64 - { - return None; - } - resolve_nan_preserving_reference(&cast.expr) - } - Expr::ScalarFunction(ScalarFunction { func, args }) - if func.name() == "abs" && args.len() == 1 => - { - resolve_nan_preserving_reference(&args[0]) - } - Expr::BinaryExpr(binary) => resolve_nan_preserving_binary(binary), - _ => None, - } -} - -/// Resolves the column reference from an arithmetic expression that combines a -/// single column with a finite literal while preserving NaN-ness. See -/// [`resolve_nan_preserving_reference`] for the soundness argument. -/// -/// Expressions with column references on both sides (e.g. `(x + 1) * (x - 2)`) -/// are not supported. Handling them safely would require both operands to -/// resolve to the *same* column (`x + y` cannot be expressed as a single -/// `col IS NAN`) and the operator combination itself to be NaN-preserving: -/// `(x + 1) * (x - 2)` is NaN iff `x` is NaN, but `(x + 1) - (x - 2)` is NaN -/// for `x = inf` (`inf - inf`) even though `x` is not. -/// -/// TODO: support NaN-preserving expressions with column references on both -/// sides, see . -fn resolve_nan_preserving_binary(binary: &BinaryExpr) -> Option { - let (left, right) = (&binary.left, &binary.right); - match binary.op { - // `x + c`, `c + x`, `x - c` and `c - x` are NaN iff `x` is NaN, for any - // finite literal `c`. The column may be on either side. - Operator::Plus | Operator::Minus => { - if finite_literal(right).is_some() { - resolve_nan_preserving_reference(left) - } else if finite_literal(left).is_some() { - resolve_nan_preserving_reference(right) - } else { - None - } - } - - // `x * c` and `c * x` are NaN iff `x` is NaN, but only when `c` is - // non-zero. Per IEEE-754: - // - inf is not NaN - // - inf * 0 is NaN - // so multiplying by zero is rejected. The column may be on either side. - Operator::Multiply => { - if matches!(finite_literal(right), Some(c) if c != 0.0) { - resolve_nan_preserving_reference(left) - } else if matches!(finite_literal(left), Some(c) if c != 0.0) { - resolve_nan_preserving_reference(right) - } else { - None - } - } - - // `x / c` is NaN iff `x` is NaN, for a finite non-zero literal `c`. - // `c / x` is rejected and the column must be the dividend (left side). - // Per IEEE-754: - // - 0 is not NaN - // - 0 / 0 is NaN - // so `c / x` is not NaN-preserving. - Operator::Divide => { - if matches!(finite_literal(right), Some(c) if c != 0.0) { - resolve_nan_preserving_reference(left) - } else { - None - } - } - - _ => None, - } -} - -/// Returns the value of `expr` as an `f64` if it is a finite numeric literal -/// (i.e. not a non-literal, non-numeric, or infinite/NaN value). The numeric -/// conversion is delegated to DataFusion's [`ScalarValue::cast_to`]; the value -/// is only used to inspect finiteness and sign (precision loss is irrelevant). -fn finite_literal(expr: &Expr) -> Option { - let Expr::Literal(value, _) = expr else { - return None; - }; - match value.cast_to(&DataType::Float64).ok()? { - ScalarValue::Float64(Some(v)) if v.is_finite() => Some(v), - _ => None, - } -} - -fn to_iceberg_and_predicate( - left: TransformedResult, - right: TransformedResult, -) -> TransformedResult { - match (left, right) { - (TransformedResult::Predicate(left), TransformedResult::Predicate(right)) => { - TransformedResult::Predicate(left.and(right)) - } - (TransformedResult::Predicate(left), _) => TransformedResult::Predicate(left), - (_, TransformedResult::Predicate(right)) => TransformedResult::Predicate(right), - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_or_predicate(left: TransformedResult, right: TransformedResult) -> TransformedResult { - match (left, right) { - (TransformedResult::Predicate(left), TransformedResult::Predicate(right)) => { - TransformedResult::Predicate(left.or(right)) - } - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_binary_predicate( - left: TransformedResult, - right: TransformedResult, - op: PredicateOperator, -) -> TransformedResult { - let (r, d, op) = match (left, right) { - (TransformedResult::NotTransformed, _) => return TransformedResult::NotTransformed, - (_, TransformedResult::NotTransformed) => return TransformedResult::NotTransformed, - (TransformedResult::Column(r), TransformedResult::Literal(d)) => (r, d, op), - (TransformedResult::Literal(d), TransformedResult::Column(r)) => { - (r, d, reverse_predicate_operator(op)) - } - _ => return TransformedResult::NotTransformed, - }; - TransformedResult::Predicate(Predicate::Binary(BinaryExpression::new(op, r, d))) -} - -fn reverse_predicate_operator(op: PredicateOperator) -> PredicateOperator { - match op { - PredicateOperator::Eq => PredicateOperator::Eq, - PredicateOperator::NotEq => PredicateOperator::NotEq, - PredicateOperator::GreaterThan => PredicateOperator::LessThan, - PredicateOperator::GreaterThanOrEq => PredicateOperator::LessThanOrEq, - PredicateOperator::LessThan => PredicateOperator::GreaterThan, - PredicateOperator::LessThanOrEq => PredicateOperator::GreaterThanOrEq, - _ => unreachable!("Reverse {}", op), - } -} - -const MILLIS_PER_DAY: i64 = 24 * 60 * 60 * 1000; - -/// Convert a scalar value to an iceberg datum. -fn scalar_value_to_datum(value: &ScalarValue) -> Option { - match value { - ScalarValue::Boolean(Some(v)) => Some(Datum::bool(*v)), - ScalarValue::Int8(Some(v)) => Some(Datum::int(*v as i32)), - ScalarValue::Int16(Some(v)) => Some(Datum::int(*v as i32)), - ScalarValue::Int32(Some(v)) => Some(Datum::int(*v)), - ScalarValue::Int64(Some(v)) => Some(Datum::long(*v)), - ScalarValue::Float32(Some(v)) => Some(Datum::double(*v as f64)), - ScalarValue::Float64(Some(v)) => Some(Datum::double(*v)), - ScalarValue::Utf8(Some(v)) => Some(Datum::string(v.clone())), - ScalarValue::LargeUtf8(Some(v)) => Some(Datum::string(v.clone())), - ScalarValue::Binary(Some(v)) => Some(Datum::binary(v.clone())), - ScalarValue::LargeBinary(Some(v)) => Some(Datum::binary(v.clone())), - ScalarValue::Date32(Some(v)) => Some(Datum::date(*v)), - ScalarValue::Date64(Some(v)) => Some(Datum::date((*v / MILLIS_PER_DAY) as i32)), - // Timestamp conversions - // Note: TimestampSecond and TimestampMillisecond are not handled here because - // DataFusion's type coercion always converts them to match the column type - // (either TimestampMicrosecond or TimestampNanosecond) before predicate pushdown. - // See unit tests for how those conversions would work if needed. - ScalarValue::TimestampMicrosecond(Some(v), _) => Some(Datum::timestamp_micros(*v)), - ScalarValue::TimestampNanosecond(Some(v), _) => Some(Datum::timestamp_nanos(*v)), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - - use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; - use datafusion::common::DFSchema; - use datafusion::logical_expr::utils::split_conjunction; - use datafusion::prelude::{Expr, SessionContext}; - use iceberg::expr::{Predicate, Reference}; - use iceberg::spec::Datum; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - - use super::convert_filters_to_predicate; - - fn create_test_schema() -> DFSchema { - let arrow_schema = Schema::new(vec![ - Field::new("foo", DataType::Int32, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("bar", DataType::Utf8, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("ts", DataType::Timestamp(TimeUnit::Second, None), true).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())]), - ), - Field::new("qux", DataType::Float64, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "4".to_string(), - )])), - ]); - DFSchema::try_from_qualified_schema("my_table", &arrow_schema).unwrap() - } - - fn convert_to_iceberg_predicate(sql: &str) -> Option { - let df_schema = create_test_schema(); - let expr = SessionContext::new() - .parse_sql_expr(sql, &df_schema) - .unwrap(); - let exprs: Vec = split_conjunction(&expr).into_iter().cloned().collect(); - convert_filters_to_predicate(&exprs[..]) - } - - #[test] - fn test_predicate_conversion_with_single_condition() { - let predicate = convert_to_iceberg_predicate("foo = 1").unwrap(); - assert_eq!(predicate, Reference::new("foo").equal_to(Datum::long(1))); - - let predicate = convert_to_iceberg_predicate("foo != 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").not_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo > 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo >= 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than_or_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo < 1").unwrap(); - assert_eq!(predicate, Reference::new("foo").less_than(Datum::long(1))); - - let predicate = convert_to_iceberg_predicate("foo <= 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").less_than_or_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo is null").unwrap(); - assert_eq!(predicate, Reference::new("foo").is_null()); - - let predicate = convert_to_iceberg_predicate("foo is not null").unwrap(); - assert_eq!(predicate, Reference::new("foo").is_not_null()); - - let predicate = convert_to_iceberg_predicate("foo in (5, 6)").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").is_in([Datum::long(5), Datum::long(6)]) - ); - - let predicate = convert_to_iceberg_predicate("foo not in (5, 6)").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").is_not_in([Datum::long(5), Datum::long(6)]) - ); - - let predicate = convert_to_iceberg_predicate("not foo = 1").unwrap(); - assert_eq!(predicate, !Reference::new("foo").equal_to(Datum::long(1))); - } - - #[test] - fn test_predicate_conversion_with_single_unsupported_condition() { - let predicate = convert_to_iceberg_predicate("foo + 1 = 1"); - assert_eq!(predicate, None); - - let predicate = convert_to_iceberg_predicate("length(bar) = 1"); - assert_eq!(predicate, None); - - let predicate = convert_to_iceberg_predicate("foo in (1, 2, foo)"); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_single_condition_rev() { - let predicate = convert_to_iceberg_predicate("1 < foo").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than(Datum::long(1)) - ); - } - - #[test] - fn test_predicate_conversion_with_and_condition() { - let sql = "foo > 1 and bar = 'test'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_and_condition_unsupported() { - let sql = "foo > 1 and length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Reference::new("foo").greater_than(Datum::long(1)); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_and_condition_both_unsupported() { - let sql = "foo in (1, 2, foo) and length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_or_condition_unsupported() { - let sql = "foo > 1 or length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_or_condition_supported() { - let sql = "foo > 1 or bar = 'test'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::or( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_complex_binary_expr() { - let sql = "(foo > 1 and bar = 'test') or foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - - let inner_predicate = Predicate::and( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - let expected_predicate = Predicate::or( - inner_predicate, - Reference::new("foo").less_than(Datum::long(0)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_one_and_expr_supported() { - let sql = "(foo > 1 and length(bar) = 1 ) or foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - - let inner_predicate = Reference::new("foo").greater_than(Datum::long(1)); - let expected_predicate = Predicate::or( - inner_predicate, - Reference::new("foo").less_than(Datum::long(0)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_complex_binary_expr_unsupported() { - let sql = "(foo > 1 or length(bar) = 1 ) and foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Reference::new("foo").less_than(Datum::long(0)); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_cast() { - let sql = "ts >= timestamp '2023-01-05T00:00:00'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = - Reference::new("ts").greater_than_or_equal_to(Datum::string("2023-01-05T00:00:00")); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_date_cast() { - let sql = "ts >= date '2023-01-05T11:00:00'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_scalar_value_to_datum_timestamp() { - use datafusion::common::ScalarValue; - - // Test TimestampMicrosecond - maps directly to Datum::timestamp_micros - let ts_micros = 1672876800000000i64; // 2023-01-05 00:00:00 UTC in microseconds - let datum = - super::scalar_value_to_datum(&ScalarValue::TimestampMicrosecond(Some(ts_micros), None)); - assert_eq!(datum, Some(Datum::timestamp_micros(ts_micros))); - - // Test TimestampNanosecond - maps to Datum::timestamp_nanos to preserve precision - let ts_nanos = 1672876800000000500i64; // 2023-01-05 00:00:00.000000500 UTC in nanoseconds - let datum = - super::scalar_value_to_datum(&ScalarValue::TimestampNanosecond(Some(ts_nanos), None)); - assert_eq!(datum, Some(Datum::timestamp_nanos(ts_nanos))); - - // Test None timestamp - let datum = super::scalar_value_to_datum(&ScalarValue::TimestampMicrosecond(None, None)); - assert_eq!(datum, None); - - // Note: TimestampSecond and TimestampMillisecond are not supported because - // DataFusion's type coercion converts them to TimestampMicrosecond or TimestampNanosecond - // before they reach scalar_value_to_datum in SQL queries. - // - // These return None (not pushed down): - let ts_seconds = 1672876800i64; // 2023-01-05 00:00:00 UTC in seconds - let datum = - super::scalar_value_to_datum(&ScalarValue::TimestampSecond(Some(ts_seconds), None)); - assert_eq!(datum, None); - - let ts_millis = 1672876800000i64; // 2023-01-05 00:00:00 UTC in milliseconds - let datum = - super::scalar_value_to_datum(&ScalarValue::TimestampMillisecond(Some(ts_millis), None)); - assert_eq!(datum, None); - } - - #[test] - fn test_scalar_value_to_datum_binary() { - use datafusion::common::ScalarValue; - - let bytes = vec![1u8, 2u8, 3u8]; - let datum = super::scalar_value_to_datum(&ScalarValue::Binary(Some(bytes.clone()))); - assert_eq!(datum, Some(Datum::binary(bytes.clone()))); - - let datum = super::scalar_value_to_datum(&ScalarValue::LargeBinary(Some(bytes.clone()))); - assert_eq!(datum, Some(Datum::binary(bytes))); - - let datum = super::scalar_value_to_datum(&ScalarValue::Binary(None)); - assert_eq!(datum, None); - } - - #[test] - fn test_predicate_conversion_with_binary() { - let sql = "foo = 1 and bar = X'0102'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - // Binary literals are converted to Datum::binary - // Note: SQL literal 1 is converted to Long by DataFusion - let expected_predicate = Reference::new("foo") - .equal_to(Datum::long(1)) - .and(Reference::new("bar").equal_to(Datum::binary(vec![1u8, 2u8]))); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_scalar_value_to_datum_boolean() { - use datafusion::common::ScalarValue; - - // Test boolean true - let datum = super::scalar_value_to_datum(&ScalarValue::Boolean(Some(true))); - assert_eq!(datum, Some(Datum::bool(true))); - - // Test boolean false - let datum = super::scalar_value_to_datum(&ScalarValue::Boolean(Some(false))); - assert_eq!(datum, Some(Datum::bool(false))); - - // Test None boolean - let datum = super::scalar_value_to_datum(&ScalarValue::Boolean(None)); - assert_eq!(datum, None); - } - - #[test] - fn test_predicate_conversion_with_like_starts_with() { - let sql = "bar LIKE 'test%'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!( - predicate, - Reference::new("bar").starts_with(Datum::string("test")) - ); - } - - #[test] - fn test_predicate_conversion_with_not_like_starts_with() { - let sql = "bar NOT LIKE 'test%'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!( - predicate, - Reference::new("bar").not_starts_with(Datum::string("test")) - ); - } - - #[test] - fn test_predicate_conversion_with_like_empty_prefix() { - let sql = "bar LIKE '%'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!( - predicate, - Reference::new("bar").starts_with(Datum::string("")) - ); - } - - #[test] - fn test_predicate_conversion_with_like_complex_pattern() { - // Patterns with wildcards in the middle cannot be pushed down - let sql = "bar LIKE 'te%st'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_like_underscore_wildcard() { - // Patterns with underscore wildcard cannot be pushed down - let sql = "bar LIKE 'test_'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_like_no_wildcard() { - // Patterns without trailing % cannot be pushed down as StartsWith - let sql = "bar LIKE 'test'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_ilike() { - // Case-insensitive LIKE (ILIKE) is not supported - let sql = "bar ILIKE 'test%'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_like_and_other_conditions() { - let sql = "bar LIKE 'test%' AND foo > 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("bar").starts_with(Datum::string("test")), - Reference::new("foo").greater_than(Datum::long(1)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_like_special_characters() { - // Test LIKE with special characters in prefix - let sql = "bar LIKE 'test-abc_123%'"; - let predicate = convert_to_iceberg_predicate(sql); - // This should not be pushed down because it contains underscore - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_like_unicode() { - // Test LIKE with unicode characters in prefix - let sql = "bar LIKE '测试%'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!( - predicate, - Reference::new("bar").starts_with(Datum::string("测试")) - ); - } - - #[test] - fn test_predicate_conversion_with_isnan() { - let predicate = convert_to_iceberg_predicate("isnan(qux)").unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_not_isnan() { - let predicate = convert_to_iceberg_predicate("NOT isnan(qux)").unwrap(); - assert_eq!(predicate, !Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_isnan_and_other_condition() { - let sql = "isnan(qux) AND foo > 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("qux").is_nan(), - Reference::new("foo").greater_than(Datum::long(1)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_isnan_negation() { - // -x is NaN iff x is NaN - let predicate = convert_to_iceberg_predicate("isnan(-qux)").unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan()); - - let predicate = convert_to_iceberg_predicate("NOT isnan(-qux)").unwrap(); - assert_eq!(predicate, !Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_isnan_abs() { - // abs(x) is NaN iff x is NaN - let predicate = convert_to_iceberg_predicate("isnan(abs(qux))").unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_isnan_additive() { - // x + c, c + x, x - c, c - x are NaN iff x is NaN (for finite c) - for sql in [ - "isnan(qux + 1)", - "isnan(1 + qux)", - "isnan(qux - 1)", - "isnan(1 - qux)", - "isnan(qux + 1.5)", - ] { - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan(), "sql: {sql}"); - } - } - - #[test] - fn test_predicate_conversion_with_isnan_multiplicative() { - // x * c, c * x, x / c are NaN iff x is NaN (for finite non-zero c) - for sql in ["isnan(qux * 2)", "isnan(2 * qux)", "isnan(qux / 2)"] { - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan(), "sql: {sql}"); - } - } - - #[test] - fn test_predicate_conversion_with_isnan_nested_expr() { - // Nested NaN-preserving transformations resolve to the inner column - let predicate = convert_to_iceberg_predicate("isnan(-(abs(qux) + 1) * 3)").unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_isnan_and_other_complex_condition() { - let sql = "isnan(qux + 1) AND foo > 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("qux").is_nan(), - Reference::new("foo").greater_than(Datum::long(1)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_isnan_unsupported_arg() { - // Multiplying/dividing by zero does not preserve NaN-ness: `x * 0` is NaN - // when `x` is ±inf, so it cannot be pushed down. - assert_eq!(convert_to_iceberg_predicate("isnan(qux * 0)"), None); - assert_eq!(convert_to_iceberg_predicate("isnan(qux / 0)"), None); - - // `c / x` is not NaN-preserving (e.g. `0 / 0` is NaN while `0` is not). - assert_eq!(convert_to_iceberg_predicate("isnan(1 / qux)"), None); - - // Expressions referencing more than one column cannot be reduced to a - // single column reference. - assert_eq!(convert_to_iceberg_predicate("isnan(qux + foo)"), None); - - // Unknown scalar functions are not pushed down. - assert_eq!(convert_to_iceberg_predicate("isnan(sqrt(qux))"), None); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs b/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs deleted file mode 100644 index 33c1143ab3..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::Arc; - -use datafusion::catalog::TableProvider; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; -use futures::TryStreamExt; - -use crate::metadata_table::IcebergMetadataTableProvider; - -#[derive(Debug)] -pub struct IcebergMetadataScan { - provider: IcebergMetadataTableProvider, - properties: Arc, -} - -impl IcebergMetadataScan { - pub fn new(provider: IcebergMetadataTableProvider) -> Self { - let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(provider.schema()), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - )); - Self { - provider, - properties, - } - } -} - -impl DisplayAs for IcebergMetadataScan { - fn fmt_as( - &self, - _t: datafusion::physical_plan::DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!(f, "IcebergMetadataScan") - } -} - -impl ExecutionPlan for IcebergMetadataScan { - fn name(&self) -> &str { - "IcebergMetadataScan" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> datafusion::error::Result, - ) -> datafusion::error::Result { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> datafusion::error::Result> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> datafusion::error::Result { - let fut = self.provider.clone().scan(); - let stream = futures::stream::once(fut).try_flatten(); - let schema = self.provider.schema(); - Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/mod.rs b/crates/integrations/datafusion/src/physical_plan/mod.rs deleted file mode 100644 index aeac30de32..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub(crate) mod commit; -pub(crate) mod expr_to_predicate; -pub(crate) mod metadata_scan; -pub(crate) mod project; -pub(crate) mod repartition; -pub(crate) mod scan; -pub(crate) mod sort; -pub(crate) mod write; - -pub(crate) const DATA_FILES_COL_NAME: &str = "data_files"; - -pub use expr_to_predicate::convert_filters_to_predicate; -pub use project::project_with_partition; -pub use scan::IcebergTableScan; diff --git a/crates/integrations/datafusion/src/physical_plan/project.rs b/crates/integrations/datafusion/src/physical_plan/project.rs deleted file mode 100644 index a77abd0eef..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/project.rs +++ /dev/null @@ -1,590 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Partition value projection for Iceberg tables. - -use std::sync::Arc; - -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema}; -use datafusion::common::{DataFusionError, Result as DFResult}; -use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_expr::expressions::Column; -use datafusion::physical_plan::projection::ProjectionExec; -use datafusion::physical_plan::{ColumnarValue, ExecutionPlan}; -use iceberg::arrow::{ - PROJECTED_PARTITION_VALUE_COLUMN, PartitionValueCalculator, schema_to_arrow_schema, - strip_metadata_from_schema, -}; -use iceberg::spec::PartitionSpec; -use iceberg::table::Table; - -use crate::to_datafusion_error; - -/// Extends an ExecutionPlan with partition value calculations for Iceberg tables. -/// -/// This function takes an input ExecutionPlan and extends it with an additional column -/// containing calculated partition values based on the table's partition specification. -/// For unpartitioned tables, returns the original plan unchanged. -/// -/// # Arguments -/// * `input` - The input ExecutionPlan to extend -/// * `table` - The Iceberg table with partition specification -/// -/// # Returns -/// * `Ok(Arc)` - Extended plan with partition values column -/// * `Err` - If partition spec is not found or transformation fails -pub fn project_with_partition( - input: Arc, - table: &Table, -) -> DFResult> { - let metadata = table.metadata(); - let partition_spec = metadata.default_partition_spec(); - let table_schema = metadata.current_schema(); - - if partition_spec.is_unpartitioned() { - return Ok(input); - } - - let input_schema = input.schema(); - - // Validate that input_schema matches the Iceberg table schema - // Strip metadata from both schemas before comparison to ignore metadata differences - let expected_arrow_schema = - schema_to_arrow_schema(table_schema.as_ref()).map_err(to_datafusion_error)?; - let input_schema_cleaned = - strip_metadata_from_schema(&input_schema).map_err(to_datafusion_error)?; - let expected_schema_cleaned = - strip_metadata_from_schema(&expected_arrow_schema).map_err(to_datafusion_error)?; - - if input_schema_cleaned != expected_schema_cleaned { - return Err(DataFusionError::Plan(format!( - "Input schema does not match Iceberg table schema.\n\ - Expected schema: {expected_schema_cleaned}\n\ - Input schema: {input_schema_cleaned}" - ))); - } - - let calculator = - PartitionValueCalculator::try_new(partition_spec.as_ref(), table_schema.as_ref()) - .map_err(to_datafusion_error)?; - - let mut projection_exprs: Vec<(Arc, String)> = - Vec::with_capacity(input_schema.fields().len() + 1); - - for (index, field) in input_schema.fields().iter().enumerate() { - let column_expr = Arc::new(Column::new(field.name(), index)); - projection_exprs.push((column_expr, field.name().clone())); - } - - let partition_expr = Arc::new(PartitionExpr::new(calculator, partition_spec.clone())); - projection_exprs.push((partition_expr, PROJECTED_PARTITION_VALUE_COLUMN.to_string())); - - let projection = ProjectionExec::try_new(projection_exprs, input)?; - Ok(Arc::new(projection)) -} - -/// PhysicalExpr implementation for partition value calculation -#[derive(Debug, Clone)] -struct PartitionExpr { - calculator: Arc, - partition_spec: Arc, -} - -impl PartitionExpr { - fn new(calculator: PartitionValueCalculator, partition_spec: Arc) -> Self { - Self { - calculator: Arc::new(calculator), - partition_spec, - } - } -} - -// Manual PartialEq/Eq implementations for pointer-based equality -// (two PartitionExpr are equal if they share the same calculator and partition_spec instances) -impl PartialEq for PartitionExpr { - fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.calculator, &other.calculator) - && Arc::ptr_eq(&self.partition_spec, &other.partition_spec) - } -} - -impl Eq for PartitionExpr {} - -impl PhysicalExpr for PartitionExpr { - fn data_type(&self, _input_schema: &ArrowSchema) -> DFResult { - Ok(self.calculator.partition_arrow_type().clone()) - } - - fn nullable(&self, _input_schema: &ArrowSchema) -> DFResult { - Ok(false) - } - - fn evaluate(&self, batch: &RecordBatch) -> DFResult { - let array = self - .calculator - .calculate(batch) - .map_err(to_datafusion_error)?; - Ok(ColumnarValue::Array(array)) - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> DFResult> { - Ok(self) - } - - fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let field_names: Vec = self - .partition_spec - .fields() - .iter() - .map(|pf| format!("{}({})", pf.transform, pf.name)) - .collect(); - write!(f, "iceberg_partition_values[{}]", field_names.join(", ")) - } -} - -impl std::fmt::Display for PartitionExpr { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let field_names: Vec<&str> = self - .partition_spec - .fields() - .iter() - .map(|pf| pf.name.as_str()) - .collect(); - write!(f, "iceberg_partition_values({})", field_names.join(", ")) - } -} - -impl std::hash::Hash for PartitionExpr { - fn hash(&self, state: &mut H) { - // Two PartitionExpr are equal if they share the same calculator and partition_spec Arcs - Arc::as_ptr(&self.calculator).hash(state); - Arc::as_ptr(&self.partition_spec).hash(state); - } -} - -#[cfg(test)] -mod tests { - use datafusion::arrow::array::{ArrayRef, Int32Array, StructArray}; - use datafusion::arrow::datatypes::{DataType, Field, Fields}; - use datafusion::physical_plan::empty::EmptyExec; - use iceberg::spec::{NestedField, PrimitiveType, Schema, StructType, Transform, Type}; - use iceberg::test_utils::test_runtime; - - use super::*; - - #[test] - fn test_partition_calculator_basic() { - let table_schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = PartitionSpec::builder(Arc::new(table_schema.clone())) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - - // Verify partition type - assert_eq!(calculator.partition_type().fields().len(), 1); - assert_eq!(calculator.partition_type().fields()[0].name, "id_partition"); - } - - #[test] - fn test_partition_expr_with_projection() { - let table_schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = Arc::new( - PartitionSpec::builder(Arc::new(table_schema.clone())) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(), - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - - let input = Arc::new(EmptyExec::new(arrow_schema.clone())); - - let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - - let mut projection_exprs: Vec<(Arc, String)> = - Vec::with_capacity(arrow_schema.fields().len() + 1); - for (i, field) in arrow_schema.fields().iter().enumerate() { - let column_expr = Arc::new(Column::new(field.name(), i)); - projection_exprs.push((column_expr, field.name().clone())); - } - - let partition_expr = Arc::new(PartitionExpr::new(calculator, partition_spec)); - projection_exprs.push((partition_expr, PROJECTED_PARTITION_VALUE_COLUMN.to_string())); - - let projection = ProjectionExec::try_new(projection_exprs, input).unwrap(); - let result = Arc::new(projection); - - assert_eq!(result.schema().fields().len(), 3); - assert_eq!(result.schema().field(0).name(), "id"); - assert_eq!(result.schema().field(1).name(), "name"); - assert_eq!(result.schema().field(2).name(), "_partition"); - } - - #[test] - fn test_partition_expr_evaluate() { - let table_schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = PartitionSpec::builder(Arc::new(table_schema.clone())) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("data", DataType::Utf8, false), - ])); - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![10, 20, 30])), - Arc::new(datafusion::arrow::array::StringArray::from(vec![ - "a", "b", "c", - ])), - ]) - .unwrap(); - - let partition_spec = Arc::new(partition_spec); - let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - let partition_type = calculator.partition_arrow_type().clone(); - let expr = PartitionExpr::new(calculator, partition_spec); - - assert_eq!(expr.data_type(&arrow_schema).unwrap(), partition_type); - assert!(!expr.nullable(&arrow_schema).unwrap()); - - let result = expr.evaluate(&batch).unwrap(); - match result { - ColumnarValue::Array(array) => { - let struct_array = array.as_any().downcast_ref::().unwrap(); - let id_partition = struct_array - .column_by_name("id_partition") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(id_partition.value(0), 10); - assert_eq!(id_partition.value(1), 20); - assert_eq!(id_partition.value(2), 30); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_nested_partition() { - let address_struct = StructType::new(vec![ - NestedField::required(3, "street", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(4, "city", Type::Primitive(PrimitiveType::String)).into(), - ]); - - let table_schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "address", Type::Struct(address_struct)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = PartitionSpec::builder(Arc::new(table_schema.clone())) - .add_partition_field("address.city", "city_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let struct_fields = Fields::from(vec![ - Field::new("street", DataType::Utf8, false), - Field::new("city", DataType::Utf8, false), - ]); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("address", DataType::Struct(struct_fields), false), - ])); - - let street_array = Arc::new(datafusion::arrow::array::StringArray::from(vec![ - "123 Main St", - "456 Oak Ave", - ])); - let city_array = Arc::new(datafusion::arrow::array::StringArray::from(vec![ - "New York", - "Los Angeles", - ])); - - let struct_array = StructArray::from(vec![ - ( - Arc::new(Field::new("street", DataType::Utf8, false)), - street_array as ArrayRef, - ), - ( - Arc::new(Field::new("city", DataType::Utf8, false)), - city_array as ArrayRef, - ), - ]); - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![1, 2])), - Arc::new(struct_array), - ]) - .unwrap(); - - let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - let array = calculator.calculate(&batch).unwrap(); - - let struct_array = array.as_any().downcast_ref::().unwrap(); - let city_partition = struct_array - .column_by_name("city_partition") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(city_partition.value(0), "New York"); - assert_eq!(city_partition.value(1), "Los Angeles"); - } - - #[test] - fn test_schema_validation_matching_schemas() { - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; - - let table_schema = Arc::new( - Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(), - ); - - let partition_spec = PartitionSpec::builder(table_schema.clone()) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = iceberg::spec::SortOrder::builder() - .build(&table_schema) - .unwrap(); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - (*table_schema).clone(), - partition_spec, - sort_order, - "/test/table".to_string(), - FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - - // Create Arrow schema matching the table schema - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let result = project_with_partition(input, &table); - assert!(result.is_ok(), "Schema validation should pass"); - } - - #[test] - fn test_schema_validation_mismatched_schemas() { - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; - - let table_schema = Arc::new( - Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(), - ); - - let partition_spec = PartitionSpec::builder(table_schema.clone()) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = iceberg::spec::SortOrder::builder() - .build(&table_schema) - .unwrap(); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - (*table_schema).clone(), - partition_spec, - sort_order, - "/test/table".to_string(), - FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - - // Create Arrow schema with different field name (mismatched) - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("different_name", DataType::Utf8, false), // Wrong field name - ])); - - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let result = project_with_partition(input, &table); - assert!( - result.is_err(), - "Schema validation should fail for mismatched schemas" - ); - assert!( - result - .unwrap_err() - .to_string() - .contains("Input schema does not match Iceberg table schema") - ); - } - - #[test] - fn test_schema_validation_with_metadata_differences() { - use std::collections::HashMap; - - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; - - let table_schema = Arc::new( - Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(), - ); - - let partition_spec = PartitionSpec::builder(table_schema.clone()) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = iceberg::spec::SortOrder::builder() - .build(&table_schema) - .unwrap(); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - (*table_schema).clone(), - partition_spec, - sort_order, - "/test/table".to_string(), - FormatVersion::V2, - HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - - // Create Arrow schema with metadata (should be ignored in comparison) - let mut metadata = HashMap::new(); - metadata.insert("extra".to_string(), "metadata".to_string()); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(metadata.clone()), - Field::new("name", DataType::Utf8, false).with_metadata(metadata), - ])); - - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let result = project_with_partition(input, &table); - assert!( - result.is_ok(), - "Schema validation should pass even with metadata differences" - ); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/repartition.rs b/crates/integrations/datafusion/src/physical_plan/repartition.rs deleted file mode 100644 index de82fab2e8..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/repartition.rs +++ /dev/null @@ -1,853 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::num::NonZeroUsize; -use std::sync::Arc; - -use datafusion::error::{DataFusionError, Result as DFResult}; -use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_plan::expressions::Column; -use datafusion::physical_plan::repartition::RepartitionExec; -use datafusion::physical_plan::{ExecutionPlan, Partitioning}; -use iceberg::arrow::PROJECTED_PARTITION_VALUE_COLUMN; -use iceberg::spec::{TableMetadata, TableMetadataRef, Transform}; -/// Creates an Iceberg-aware repartition execution plan that optimizes data distribution -/// for parallel processing while respecting Iceberg table partitioning semantics. -/// -/// Automatically determines the optimal partitioning strategy based on the table's -/// partition specification. -/// -/// ## Partitioning Strategies -/// -/// - **Partitioned tables with Identity/Bucket transforms** – Uses hash partitioning on the -/// `_partition` column for optimal data distribution and file clustering. Ensures that rows -/// with the same partition values are co-located in the same task. -/// -/// - **Partitioned tables with temporal transforms** – Uses round-robin partitioning for -/// temporal transforms (Year, Month, Day, Hour) that don't provide uniform hash distribution. -/// -/// - **Unpartitioned tables** – Uses round-robin distribution to balance load evenly across workers. -/// -/// ## Requirements -/// -/// - **For partitioned tables**: The input MUST include the `_partition` column. -/// Add it by calling [`project_with_partition`](crate::physical_plan::project_with_partition) before [`repartition`]. -/// - **For unpartitioned tables**: No special preparation needed. -/// - Returns an error if a partitioned table is missing the `_partition` column. -/// -/// ## Performance Notes -/// -/// - Only adds repartitioning when the input partitioning differs from the target. -/// - Requires an explicit target partition count for deterministic behavior. -/// -/// # Arguments -/// -/// * `input` - The input [`ExecutionPlan`]. For partitioned tables, must include the `_partition` -/// column (added via [`project_with_partition`](crate::physical_plan::project_with_partition)). -/// * `table_metadata` - Iceberg table metadata containing partition spec. -/// * `target_partitions` - Target number of partitions for parallel processing (must be > 0). -/// -/// # Returns -/// -/// An [`ExecutionPlan`] that applies the optimal partitioning strategy, or the original input plan -/// if repartitioning is not needed. -/// -/// # Errors -/// -/// Returns [`DataFusionError::Plan`] if a partitioned table input is missing the `_partition` column. -/// -/// # Examples -/// -/// For partitioned tables, first add the `_partition` column: -/// -/// ```ignore -/// use std::num::NonZeroUsize; -/// use iceberg_datafusion::physical_plan::project_with_partition; -/// -/// let plan_with_partition = project_with_partition(input_plan, &table)?; -/// -/// let repartitioned_plan = repartition( -/// plan_with_partition, -/// table.metadata_ref(), -/// NonZeroUsize::new(4).unwrap(), -/// )?; -/// ``` -pub(crate) fn repartition( - input: Arc, - table_metadata: TableMetadataRef, - target_partitions: NonZeroUsize, -) -> DFResult> { - let partitioning_strategy = - determine_partitioning_strategy(&input, &table_metadata, target_partitions)?; - - Ok(Arc::new(RepartitionExec::try_new( - input, - partitioning_strategy, - )?)) -} - -/// Determine the optimal partitioning strategy based on table metadata. -/// -/// Analyzes the table's partition specification to select the most appropriate -/// DataFusion partitioning strategy for insert operations. -/// -/// ## Partitioning Strategy -/// -/// - **Partitioned tables**: Must have the `_partition` column in the input schema (added via -/// `project_with_partition`). Uses hash partitioning if the partition spec contains Identity -/// or Bucket transforms for good data distribution. Falls back to round-robin for temporal -/// transforms (Year, Month, Day, Hour) that don't provide uniform hash distribution. -/// -/// - **Unpartitioned tables**: Always uses round-robin batch partitioning to ensure even load -/// distribution across workers. -/// -/// ## Requirements -/// -/// - **For partitioned tables**: The input MUST include the `_partition` column -/// (added via `project_with_partition()`). -/// - **For unpartitioned tables**: No special preparation needed. -/// - Returns an error if a partitioned table is missing the `_partition` column. -fn determine_partitioning_strategy( - input: &Arc, - table_metadata: &TableMetadata, - target_partitions: NonZeroUsize, -) -> DFResult { - let partition_spec = table_metadata.default_partition_spec(); - let input_schema = input.schema(); - let target_partition_count = target_partitions.get(); - - // Check if partition spec has transforms suitable for hash partitioning - let has_hash_friendly_transforms = partition_spec - .fields() - .iter() - .any(|pf| matches!(pf.transform, Transform::Identity | Transform::Bucket(_))); - - let partition_col_result = input_schema.index_of(PROJECTED_PARTITION_VALUE_COLUMN); - let is_partitioned_table = !partition_spec.is_unpartitioned(); - - match (is_partitioned_table, partition_col_result) { - // Case 1: Partitioned table with _partition column present - (true, Ok(partition_col_idx)) => { - let partition_expr = Arc::new(Column::new( - PROJECTED_PARTITION_VALUE_COLUMN, - partition_col_idx, - )) as Arc; - - if has_hash_friendly_transforms { - Ok(Partitioning::Hash( - vec![partition_expr], - target_partition_count, - )) - } else { - Ok(Partitioning::RoundRobinBatch(target_partition_count)) - } - } - - // Case 2: Partitioned table missing _partition column (normally this should not happen) - (true, Err(_)) => Err(DataFusionError::Plan(format!( - "Partitioned table input missing {PROJECTED_PARTITION_VALUE_COLUMN} column. \ - Ensure projection happens before repartitioning." - ))), - - // Case 3: Unpartitioned table, always use RoundRobinBatch - (false, _) => Ok(Partitioning::RoundRobinBatch(target_partition_count)), - } -} - -#[cfg(test)] -mod tests { - use datafusion::arrow::datatypes::{ - DataType as ArrowDataType, Field as ArrowField, Fields, Schema as ArrowSchema, TimeUnit, - }; - use datafusion::execution::TaskContext; - use datafusion::physical_plan::empty::EmptyExec; - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::spec::{ - NestedField, NullOrder, PrimitiveType, Schema, SortDirection, SortField, SortOrder, - Transform, Type, - }; - use iceberg::table::Table; - use iceberg::test_utils::test_runtime; - - use super::*; - - fn create_test_table() -> Table { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 2, - "data", - Type::Primitive(PrimitiveType::String), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .build() - .unwrap(); - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - - Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/metadata.json") - .runtime(test_runtime()) - .build() - .unwrap() - } - - fn create_test_arrow_schema() -> Arc { - Arc::new(ArrowSchema::new(vec![ - ArrowField::new("id", ArrowDataType::Int64, false), - ArrowField::new("data", ArrowDataType::Utf8, false), - ])) - } - - #[tokio::test] - async fn test_repartition_unpartitioned_table() { - let table = create_test_table(); - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let repartitioned_plan = repartition( - input.clone(), - table.metadata_ref(), - NonZeroUsize::new(4).unwrap(), - ) - .unwrap(); - - assert_ne!(input.name(), repartitioned_plan.name()); - assert_eq!(repartitioned_plan.name(), "RepartitionExec"); - } - - #[tokio::test] - async fn test_repartition_explicit_partitions() { - let table = create_test_table(); - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(8).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::RoundRobinBatch(n) => { - assert_eq!(*n, 8); - } - _ => panic!("Expected RoundRobinBatch partitioning"), - } - } - - #[tokio::test] - async fn test_repartition_zero_partitions_fails() { - let _table = create_test_table(); - let _input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let result = NonZeroUsize::new(0); - assert!(result.is_none(), "NonZeroUsize::new(0) should return None"); - - // Test that we can't call repartition with 0 partitions - // This is prevented at compile time by NonZeroUsize - let _ = result; // This would be None, so we can't call repartition - } - - #[tokio::test] - async fn test_partition_count_validation() { - let table = create_test_table(); - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let target_partitions = 16; - let repartitioned_plan = repartition( - input, - table.metadata_ref(), - NonZeroUsize::new(target_partitions).unwrap(), - ) - .unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::RoundRobinBatch(n) => { - assert_eq!(*n, target_partitions); - } - _ => panic!("Expected RoundRobinBatch partitioning"), - } - } - - #[tokio::test] - async fn test_datafusion_repartitioning_integration() { - let table = create_test_table(); - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(3).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::RoundRobinBatch(n) => { - assert_eq!(*n, 3, "Should use round-robin for unpartitioned table"); - } - _ => panic!("Expected RoundRobinBatch partitioning for unpartitioned table"), - } - - let task_ctx = Arc::new(TaskContext::default()); - let stream = repartitioned_plan.execute(0, task_ctx.clone()).unwrap(); - - assert!(!stream.schema().fields().is_empty()); - } - - #[tokio::test] - async fn test_bucket_aware_partitioning() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 2, - "category", - Type::Primitive(PrimitiveType::String), - )), - ]) - .build() - .unwrap(); - - let sort_order = SortOrder::builder() - .with_order_id(1) - .with_sort_field(SortField { - source_id: 2, - transform: Transform::Bucket(4), - direction: SortDirection::Ascending, - null_order: NullOrder::First, - }) - .build(&schema) - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .build() - .unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/bucketed_table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "bucketed_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/bucketed_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("id", ArrowDataType::Int64, false), - ArrowField::new("category", ArrowDataType::Utf8, false), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - // For bucketed tables without _partition column, should use round-robin - // since the new logic prioritizes _partition column when available - match partitioning { - Partitioning::Hash(_, _) => { - // This would happen if _partition column is present - } - Partitioning::RoundRobinBatch(_) => { - // This happens when _partition column is not present - } - _ => panic!("Unexpected partitioning strategy"), - } - } - - #[tokio::test] - async fn test_combined_partition_and_bucket_strategy() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "date", - Type::Primitive(PrimitiveType::Date), - )), - Arc::new(NestedField::required( - 2, - "user_id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 3, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("date", "date", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder() - .with_order_id(1) - .with_sort_field(SortField { - source_id: 2, - transform: Transform::Bucket(8), - direction: SortDirection::Ascending, - null_order: NullOrder::First, - }) - .build(&schema) - .unwrap(); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/partitioned_bucketed_table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "partitioned_bucketed_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/partitioned_bucketed_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("date", ArrowDataType::Date32, false), - ArrowField::new("user_id", ArrowDataType::Int64, false), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::Hash(exprs, _) => { - // Should use _partition column for hash partitioning - assert_eq!( - exprs.len(), - 1, - "Should have exactly one hash column (_partition)" - ); - - let column_names: Vec = exprs - .iter() - .filter_map(|expr| { - expr.downcast_ref::() - .map(|col| col.name().to_string()) - }) - .collect(); - - assert!( - column_names.contains(&PROJECTED_PARTITION_VALUE_COLUMN.to_string()), - "Should use _partition column, got: {column_names:?}" - ); - } - _ => panic!("Expected Hash partitioning with Identity transform"), - } - } - - #[tokio::test] - async fn test_none_distribution_mode_fallback() { - let schema = Schema::builder() - .with_fields(vec![Arc::new(NestedField::required( - 1, - "id", - Type::Primitive(PrimitiveType::Long), - ))]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .build() - .unwrap(); - let sort_order = SortOrder::builder().build(&schema).unwrap(); - - let mut properties = std::collections::HashMap::new(); - properties.insert("write.distribution-mode".to_string(), "none".to_string()); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/none_table".to_string(), - iceberg::spec::FormatVersion::V2, - properties, - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "none_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/none_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - assert!( - matches!(partitioning, Partitioning::RoundRobinBatch(_)), - "Should use round-robin for 'none' distribution mode" - ); - } - - #[tokio::test] - async fn test_schema_ref_convenience_method() { - let table = create_test_table(); - - let schema_ref_1 = table.current_schema_ref(); - let schema_ref_2 = Arc::clone(table.metadata().current_schema()); - - assert!( - Arc::ptr_eq(&schema_ref_1, &schema_ref_2), - "current_schema_ref() should return the same Arc as manual approach" - ); - } - - #[tokio::test] - async fn test_range_only_partitions_use_round_robin() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "date", - Type::Primitive(PrimitiveType::Date), - )), - Arc::new(NestedField::required( - 2, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("date", "date_day", Transform::Day) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/range_only_table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "range_only_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/range_only_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("date", ArrowDataType::Date32, false), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - assert!( - matches!(partitioning, Partitioning::RoundRobinBatch(_)), - "Should use round-robin for temporal transforms (Day) that don't provide good hash distribution" - ); - } - - #[tokio::test] - async fn test_mixed_transforms_use_hash_partitioning() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "date", - Type::Primitive(PrimitiveType::Date), - )), - Arc::new(NestedField::required( - 2, - "user_id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 3, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("date", "date_day", Transform::Day) - .unwrap() - .add_partition_field("user_id", "user_id", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/mixed_transforms_table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "mixed_transforms_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/mixed_transforms_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("date", ArrowDataType::Date32, false), - ArrowField::new("user_id", ArrowDataType::Int64, false), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::Hash(exprs, _) => { - assert_eq!(exprs.len(), 1, "Should have one hash column (_partition)"); - let column_names: Vec = exprs - .iter() - .filter_map(|expr| { - expr.downcast_ref::() - .map(|col| col.name().to_string()) - }) - .collect(); - assert!( - column_names.contains(&PROJECTED_PARTITION_VALUE_COLUMN.to_string()), - "Should use _partition column for mixed transforms with Identity, got: {column_names:?}" - ); - } - _ => panic!("Expected Hash partitioning for table with identity transforms"), - } - } - - #[tokio::test] - async fn test_partition_column_with_temporal_transforms_uses_round_robin() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "event_time", - Type::Primitive(PrimitiveType::Timestamp), - )), - Arc::new(NestedField::required( - 2, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("event_time", "event_month", Transform::Month) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/temporal_partition".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "temporal_partition"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/temporal_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new( - "event_time", - ArrowDataType::Timestamp(TimeUnit::Microsecond, None), - false, - ), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - assert!( - matches!(partitioning, Partitioning::RoundRobinBatch(_)), - "Should use round-robin for _partition column with temporal transforms, not Hash" - ); - } - - #[tokio::test] - async fn test_partition_column_with_identity_transforms_uses_hash() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "user_id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 2, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("user_id", "user_id", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/identity_partition".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "identity_partition"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/identity_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("user_id", ArrowDataType::Int64, false), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - assert!( - matches!(partitioning, Partitioning::Hash(_, _)), - "Should use Hash partitioning for _partition column with Identity transforms" - ); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs deleted file mode 100644 index 1796304552..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ /dev/null @@ -1,252 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::pin::Pin; -use std::sync::Arc; -use std::vec; - -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::error::Result as DFResult; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; -use datafusion::prelude::Expr; -use futures::{Stream, TryStreamExt}; -use iceberg::expr::Predicate; -use iceberg::table::Table; - -use super::expr_to_predicate::convert_filters_to_predicate; -use crate::to_datafusion_error; - -/// Manages the scanning process of an Iceberg [`Table`], encapsulating the -/// necessary details and computed properties required for execution planning. -#[derive(Debug)] -pub struct IcebergTableScan { - /// A table in the catalog. - table: Table, - /// Snapshot of the table to scan. - snapshot_id: Option, - /// Stores certain, often expensive to compute, - /// plan properties used in query optimization. - plan_properties: Arc, - /// Projection column names, None means all columns - projection: Option>, - /// Filters to apply to the table scan - predicates: Option, - /// Optional limit on the number of rows to return - limit: Option, -} - -impl IcebergTableScan { - /// Creates a new [`IcebergTableScan`] object. - pub(crate) fn new( - table: Table, - snapshot_id: Option, - schema: ArrowSchemaRef, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> Self { - let output_schema = match projection { - None => schema.clone(), - Some(projection) => Arc::new(schema.project(projection).unwrap()), - }; - let plan_properties = Self::compute_properties(output_schema.clone()); - let projection = get_column_names(schema.clone(), projection); - let predicates = convert_filters_to_predicate(filters); - - Self { - table, - snapshot_id, - plan_properties, - projection, - predicates, - limit, - } - } - - pub fn table(&self) -> &Table { - &self.table - } - - pub fn snapshot_id(&self) -> Option { - self.snapshot_id - } - - pub fn projection(&self) -> Option<&[String]> { - self.projection.as_deref() - } - - pub fn predicates(&self) -> Option<&Predicate> { - self.predicates.as_ref() - } - - pub fn limit(&self) -> Option { - self.limit - } - - /// Computes [`PlanProperties`] used in query optimization. - fn compute_properties(schema: ArrowSchemaRef) -> Arc { - // TODO: - // This is more or less a placeholder, to be replaced - // once we support output-partitioning - Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - )) - } -} - -impl ExecutionPlan for IcebergTableScan { - fn name(&self) -> &str { - "IcebergTableScan" - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> DFResult> { - Ok(self) - } - - fn properties(&self) -> &Arc { - &self.plan_properties - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> DFResult { - let fut = get_batch_stream( - self.table.clone(), - self.snapshot_id, - self.projection.clone(), - self.predicates.clone(), - ); - let stream = futures::stream::once(fut).try_flatten(); - - // Apply limit if specified - let limited_stream: Pin> + Send>> = - if let Some(limit) = self.limit { - let mut remaining = limit; - Box::pin(stream.try_filter_map(move |batch| { - futures::future::ready(if remaining == 0 { - Ok(None) - } else if batch.num_rows() <= remaining { - remaining -= batch.num_rows(); - Ok(Some(batch)) - } else { - let limited_batch = batch.slice(0, remaining); - remaining = 0; - Ok(Some(limited_batch)) - }) - })) - } else { - Box::pin(stream) - }; - - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - limited_stream, - ))) - } -} - -impl DisplayAs for IcebergTableScan { - fn fmt_as( - &self, - _t: datafusion::physical_plan::DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!( - f, - "IcebergTableScan projection:[{}] predicate:[{}]", - self.projection - .clone() - .map_or(String::new(), |v| v.join(",")), - self.predicates - .clone() - .map_or(String::from(""), |p| format!("{p}")) - )?; - if let Some(limit) = self.limit { - write!(f, " limit:[{limit}]")?; - } - Ok(()) - } -} - -/// Asynchronously retrieves a stream of [`RecordBatch`] instances -/// from a given table. -/// -/// This function initializes a [`TableScan`], builds it, -/// and then converts it into a stream of Arrow [`RecordBatch`]es. -async fn get_batch_stream( - table: Table, - snapshot_id: Option, - column_names: Option>, - predicates: Option, -) -> DFResult> + Send>>> { - let scan_builder = match snapshot_id { - Some(snapshot_id) => table.scan().snapshot_id(snapshot_id), - None => table.scan(), - }; - - let mut scan_builder = match column_names { - Some(column_names) => scan_builder.select(column_names), - None => scan_builder.select_all(), - }; - if let Some(pred) = predicates { - scan_builder = scan_builder.with_filter(pred); - } - let table_scan = scan_builder.build().map_err(to_datafusion_error)?; - - let stream = table_scan - .to_arrow() - .await - .map_err(to_datafusion_error)? - .map_err(to_datafusion_error); - Ok(Box::pin(stream)) -} - -fn get_column_names( - schema: ArrowSchemaRef, - projection: Option<&Vec>, -) -> Option> { - projection.map(|v| { - v.iter() - .map(|p| schema.field(*p).name().clone()) - .collect::>() - }) -} diff --git a/crates/integrations/datafusion/src/physical_plan/sort.rs b/crates/integrations/datafusion/src/physical_plan/sort.rs deleted file mode 100644 index 587ab120ca..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/sort.rs +++ /dev/null @@ -1,240 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Partition-based sorting for Iceberg tables. - -use std::sync::Arc; - -use datafusion::arrow::compute::SortOptions; -use datafusion::common::Result as DFResult; -use datafusion::error::DataFusionError; -use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; -use datafusion::physical_plan::ExecutionPlan; -use datafusion::physical_plan::expressions::Column; -use datafusion::physical_plan::sorts::sort::SortExec; -use iceberg::arrow::PROJECTED_PARTITION_VALUE_COLUMN; - -/// Sorts an ExecutionPlan by partition values for Iceberg tables. -/// -/// This function takes an input ExecutionPlan that has been extended with partition values -/// (via `project_with_partition`) and returns a SortExec that sorts by the partition column. -/// The partition values are expected to be in a struct column named `PROJECTED_PARTITION_VALUE_COLUMN`. -/// -/// For unpartitioned tables or plans without the partition column, returns an error. -/// -/// # Arguments -/// * `input` - The input ExecutionPlan with projected partition values -/// -/// # Returns -/// * `Ok(Arc)` - A SortExec that sorts by partition values -/// * `Err` - If the partition column is not found -pub(crate) fn sort_by_partition(input: Arc) -> DFResult> { - let schema = input.schema(); - - // Find the partition column in the schema - let (partition_column_index, _partition_field) = schema - .column_with_name(PROJECTED_PARTITION_VALUE_COLUMN) - .ok_or_else(|| { - DataFusionError::Plan(format!( - "Partition column '{PROJECTED_PARTITION_VALUE_COLUMN}' not found in schema. Ensure the plan has been extended with partition values using project_with_partition." - )) - })?; - - // Create a single sort expression for the partition column - let column_expr = Arc::new(Column::new( - PROJECTED_PARTITION_VALUE_COLUMN, - partition_column_index, - )); - - let sort_expr = PhysicalSortExpr { - expr: column_expr, - options: SortOptions::default(), // Ascending, nulls last - }; - - // Create a SortExec with preserve_partitioning=true to ensure the output partitioning - // is the same as the input partitioning, and the data is sorted within each partition - let lex_ordering = LexOrdering::new(vec![sort_expr]).ok_or_else(|| { - DataFusionError::Plan("Failed to create LexOrdering from sort expression".to_string()) - })?; - - let sort_exec = SortExec::new(lex_ordering, input).with_preserve_partitioning(true); - - Ok(Arc::new(sort_exec)) -} - -#[cfg(test)] -mod tests { - use datafusion::arrow::array::{Int32Array, RecordBatch, StringArray, StructArray}; - use datafusion::arrow::datatypes::{DataType, Field, Fields, Schema as ArrowSchema}; - use datafusion::datasource::{MemTable, TableProvider}; - use datafusion::prelude::SessionContext; - - use super::*; - - #[tokio::test] - async fn test_sort_by_partition_basic() { - // Create a schema with a partition column - let partition_fields = - Fields::from(vec![Field::new("id_partition", DataType::Int32, false)]); - - let schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - Field::new( - PROJECTED_PARTITION_VALUE_COLUMN, - DataType::Struct(partition_fields.clone()), - false, - ), - ])); - - // Create test data with partition values - let id_array = Arc::new(Int32Array::from(vec![3, 1, 2])); - let name_array = Arc::new(StringArray::from(vec!["c", "a", "b"])); - let partition_array = Arc::new(StructArray::from(vec![( - Arc::new(Field::new("id_partition", DataType::Int32, false)), - Arc::new(Int32Array::from(vec![3, 1, 2])) as _, - )])); - - let batch = - RecordBatch::try_new(schema.clone(), vec![id_array, name_array, partition_array]) - .unwrap(); - - let ctx = SessionContext::new(); - let mem_table = MemTable::try_new(schema.clone(), vec![vec![batch]]).unwrap(); - let input = mem_table.scan(&ctx.state(), None, &[], None).await.unwrap(); - - // Apply sort - let sorted_plan = sort_by_partition(input).unwrap(); - - // Execute and verify - let result = datafusion::physical_plan::collect(sorted_plan, ctx.task_ctx()) - .await - .unwrap(); - - assert_eq!(result.len(), 1); - let result_batch = &result[0]; - - let id_col = result_batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - - // Verify data is sorted by partition value - assert_eq!(id_col.value(0), 1); - assert_eq!(id_col.value(1), 2); - assert_eq!(id_col.value(2), 3); - } - - #[tokio::test] - async fn test_sort_by_partition_missing_column() { - let schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - - let batch = RecordBatch::try_new(schema.clone(), vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["a", "b", "c"])), - ]) - .unwrap(); - - let ctx = SessionContext::new(); - let mem_table = MemTable::try_new(schema.clone(), vec![vec![batch]]).unwrap(); - let input = mem_table.scan(&ctx.state(), None, &[], None).await.unwrap(); - - let result = sort_by_partition(input); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("Partition column '_partition' not found") - ); - } - - #[tokio::test] - async fn test_sort_by_partition_multi_field() { - // Test with multiple partition fields in the struct - let partition_fields = Fields::from(vec![ - Field::new("year", DataType::Int32, false), - Field::new("month", DataType::Int32, false), - ]); - - let schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("data", DataType::Utf8, false), - Field::new( - PROJECTED_PARTITION_VALUE_COLUMN, - DataType::Struct(partition_fields.clone()), - false, - ), - ])); - - // Create test data with partition values (year, month) - let id_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); - let data_array = Arc::new(StringArray::from(vec!["a", "b", "c", "d"])); - - // Partition values: (2024, 2), (2024, 1), (2023, 12), (2024, 1) - let year_array = Arc::new(Int32Array::from(vec![2024, 2024, 2023, 2024])); - let month_array = Arc::new(Int32Array::from(vec![2, 1, 12, 1])); - - let partition_array = Arc::new(StructArray::from(vec![ - ( - Arc::new(Field::new("year", DataType::Int32, false)), - year_array as _, - ), - ( - Arc::new(Field::new("month", DataType::Int32, false)), - month_array as _, - ), - ])); - - let batch = - RecordBatch::try_new(schema.clone(), vec![id_array, data_array, partition_array]) - .unwrap(); - - let ctx = SessionContext::new(); - let mem_table = MemTable::try_new(schema.clone(), vec![vec![batch]]).unwrap(); - let input = mem_table.scan(&ctx.state(), None, &[], None).await.unwrap(); - - // Apply sort - let sorted_plan = sort_by_partition(input).unwrap(); - - // Execute and verify - let result = datafusion::physical_plan::collect(sorted_plan, ctx.task_ctx()) - .await - .unwrap(); - - assert_eq!(result.len(), 1); - let result_batch = &result[0]; - - let id_col = result_batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - - // Verify data is sorted by partition value (struct comparison) - // Expected order: (2023, 12), (2024, 1), (2024, 1), (2024, 2) - // Which corresponds to ids: 3, 2, 4, 1 - assert_eq!(id_col.value(0), 3); - assert_eq!(id_col.value(1), 2); - assert_eq!(id_col.value(2), 4); - assert_eq!(id_col.value(3), 1); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs deleted file mode 100644 index 40ad9fad00..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ /dev/null @@ -1,653 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fmt::{Debug, Formatter}; -use std::str::FromStr; -use std::sync::Arc; - -use datafusion::arrow::array::{ArrayRef, RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, -}; -use datafusion::common::Result as DFResult; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::error::DataFusionError; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, - execute_input_stream, -}; -use futures::StreamExt; -use iceberg::arrow::FieldMatchMode; -use iceberg::spec::{DataFileFormat, serialize_data_file_to_json}; -use iceberg::table::Table; -use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -use iceberg::writer::file_writer::ParquetWriterBuilder; -use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, -}; -use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; -use iceberg::{Error, ErrorKind}; -use uuid::Uuid; - -use crate::physical_plan::DATA_FILES_COL_NAME; -use crate::task_writer::TaskWriter; -use crate::to_datafusion_error; - -/// An execution plan node that writes data to an Iceberg table. -/// -/// This execution plan takes input data from a child execution plan and writes it to an Iceberg table. -/// It handles the creation of data files in the appropriate format and returns information about the written files as its output. -/// -/// The output of this execution plan is a record batch containing a single column with serialized -/// data file information that can be used for committing the write operation to the table. -#[derive(Debug)] -pub(crate) struct IcebergWriteExec { - table: Table, - input: Arc, - result_schema: ArrowSchemaRef, - plan_properties: Arc, -} - -impl IcebergWriteExec { - pub fn new(table: Table, input: Arc) -> Self { - let plan_properties = Self::compute_properties(&input, Self::make_result_schema()); - - Self { - table, - input, - result_schema: Self::make_result_schema(), - plan_properties, - } - } - - fn compute_properties( - input: &Arc, - schema: ArrowSchemaRef, - ) -> Arc { - Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(input.output_partitioning().partition_count()), - EmissionType::Final, - Boundedness::Bounded, - )) - } - - // Create a record batch with serialized data files - fn make_result_batch(data_files: Vec) -> DFResult { - let files_array = Arc::new(StringArray::from(data_files)) as ArrayRef; - - RecordBatch::try_new(Self::make_result_schema(), vec![files_array]).map_err(|e| { - DataFusionError::ArrowError( - Box::new(e), - Some("Failed to make result batch".to_string()), - ) - }) - } - - fn make_result_schema() -> ArrowSchemaRef { - // Define a schema. - Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])) - } -} - -impl DisplayAs for IcebergWriteExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default => { - write!(f, "IcebergWriteExec: table={}", self.table.identifier()) - } - DisplayFormatType::Verbose => { - write!( - f, - "IcebergWriteExec: table={}, result_schema={:?}", - self.table.identifier(), - self.result_schema - ) - } - DisplayFormatType::TreeRender => { - write!(f, "IcebergWriteExec: table={}", self.table.identifier()) - } - } - } -} - -impl ExecutionPlan for IcebergWriteExec { - fn name(&self) -> &str { - "IcebergWriteExec" - } - - /// Prevents the introduction of additional `RepartitionExec` and processing input in parallel. - fn benefits_from_input_partitioning(&self) -> Vec { - vec![false] - } - - fn maintains_input_order(&self) -> Vec { - // Maintains ordering in the sense that the written file will reflect the ordering of the input. - vec![true; self.children().len()] - } - - fn properties(&self) -> &Arc { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.input] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> DFResult> { - if children.len() != 1 { - return Err(DataFusionError::Internal(format!( - "IcebergWriteExec expects exactly one child, but provided {}", - children.len() - ))); - } - - Ok(Arc::new(Self::new( - self.table.clone(), - Arc::clone(&children[0]), - ))) - } - - /// Executes the write operation for the given partition. - /// - /// This function: - /// 1. Sets up a data file writer based on the table's configuration - /// 2. Processes input data from the child execution plan - /// 3. Writes the data to files using the configured writer - /// 4. Returns a stream containing information about the written data files - /// - /// The output of this function is a stream of record batches with the following structure: - /// - /// ```text - /// +------------------+ - /// | data_files | - /// +------------------+ - /// | "{"file_path":.. | <- JSON string representing a data file - /// +------------------+ - /// ``` - /// - /// Each row in the output contains a JSON string representing a data file that was written. - /// - /// This output can be used by a subsequent operation to commit the added files to the table. - fn execute( - &self, - partition: usize, - context: Arc, - ) -> DFResult { - let partition_type = self.table.metadata().default_partition_type().clone(); - let format_version = self.table.metadata().format_version(); - - // Get typed table properties - let table_props = self.table.metadata().table_properties(); - - // Check data file format - let write_format_default = table_props - .write_format_default() - .map_err(to_datafusion_error)?; - let file_format = - DataFileFormat::from_str(&write_format_default).map_err(to_datafusion_error)?; - if file_format != DataFileFormat::Parquet { - return Err(to_datafusion_error(Error::new( - ErrorKind::FeatureUnsupported, - format!("File format {file_format} is not supported for insert_into yet!"), - ))); - } - - // Build the writer from the already-parsed table properties so it honors - // `write.parquet.*` settings (e.g. CDC). Arrow batches flowing through - // DataFusion carry no field-id metadata, so match fields by name. - let mut parquet_file_writer_builder = ParquetWriterBuilder::from_table_properties( - &table_props, - self.table.metadata().current_schema().clone(), - ) - .map_err(to_datafusion_error)? - .with_match_mode(FieldMatchMode::Name); - if let Some(encryption_manager) = self.table.encryption_manager() { - parquet_file_writer_builder = - parquet_file_writer_builder.with_encryption_manager(encryption_manager.clone()); - } - let target_file_size = table_props - .write_target_file_size_bytes() - .map_err(to_datafusion_error)?; - - let file_io = self.table.file_io().clone(); - // todo location_gen and file_name_gen should be configurable - let location_generator = - DefaultLocationGenerator::new(self.table.metadata()).map_err(to_datafusion_error)?; - // todo filename prefix/suffix should be configurable - let file_name_generator = - DefaultFileNameGenerator::new(Uuid::now_v7().to_string(), None, file_format); - let rolling_writer_builder = RollingFileWriterBuilder::new( - parquet_file_writer_builder, - target_file_size, - file_io, - location_generator, - file_name_generator, - ); - let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); - - // Create TaskWriter - let fanout_enabled = table_props - .write_datafusion_fanout_enabled() - .map_err(to_datafusion_error)?; - let schema = self.table.metadata().current_schema().clone(); - let partition_spec = self.table.metadata().default_partition_spec().clone(); - let task_writer = TaskWriter::try_new( - data_file_writer_builder, - fanout_enabled, - schema.clone(), - partition_spec, - ) - .map_err(to_datafusion_error)?; - - // Get input data - let data = execute_input_stream( - Arc::clone(&self.input), - self.input.schema(), // input schema may have projected column `_partition` - partition, - Arc::clone(&context), - )?; - - // Create write stream - let stream = futures::stream::once(async move { - let mut task_writer = task_writer; - let mut input_stream = data; - - while let Some(batch) = input_stream.next().await { - let batch = batch?; - task_writer - .write(batch) - .await - .map_err(to_datafusion_error)?; - } - - let data_files = task_writer.close().await.map_err(to_datafusion_error)?; - - // Convert builders to data files and then to JSON strings - let data_files_strs: Vec = data_files - .into_iter() - .map(|data_file| { - serialize_data_file_to_json(data_file, &partition_type, format_version) - .map_err(to_datafusion_error) - }) - .collect::>>()?; - - Self::make_result_batch(data_files_strs) - }) - .boxed(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.result_schema), - stream, - ))) - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::fmt::{Debug, Formatter}; - use std::sync::Arc; - - use datafusion::arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray}; - use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, - }; - use datafusion::common::Result as DFResult; - use datafusion::execution::{SendableRecordBatchStream, TaskContext}; - use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; - use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; - use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; - use futures::{StreamExt, stream}; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::spec::{ - DataFileFormat, NestedField, PrimitiveType, Schema, Type, deserialize_data_file_from_json, - }; - use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, Result, TableCreation}; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - use tempfile::TempDir; - - use super::*; - - /// A simple execution plan that returns a predefined set of record batches - struct MockExecutionPlan { - schema: ArrowSchemaRef, - batches: Vec, - properties: Arc, - } - - impl MockExecutionPlan { - fn new(schema: ArrowSchemaRef, batches: Vec) -> Self { - let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema.clone()), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - )); - - Self { - schema, - batches, - properties, - } - } - } - - impl Debug for MockExecutionPlan { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "MockExecutionPlan") - } - } - - impl DisplayAs for MockExecutionPlan { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default - | DisplayFormatType::Verbose - | DisplayFormatType::TreeRender => { - write!(f, "MockExecutionPlan") - } - } - } - } - - impl ExecutionPlan for MockExecutionPlan { - fn name(&self) -> &str { - "MockExecutionPlan" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> DFResult> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> DFResult { - let batches = self.batches.clone(); - let stream = stream::iter(batches.into_iter().map(Ok)); - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema.clone(), - stream.boxed(), - ))) - } - } - - /// Helper function to create a temporary directory and return its path - fn temp_path() -> String { - let temp_dir = TempDir::new().unwrap(); - temp_dir.path().to_str().unwrap().to_string() - } - - /// Helper function to create a memory catalog - async fn get_iceberg_catalog() -> MemoryCatalog { - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), temp_path())]), - ) - .await - .unwrap() - } - - /// Helper function to create a test table schema - fn get_test_schema() -> Result { - Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - } - - /// Helper function to create a table creation - fn get_table_creation( - location: impl ToString, - name: impl ToString, - schema: Schema, - ) -> TableCreation { - TableCreation::builder() - .location(location.to_string()) - .name(name.to_string()) - .properties(HashMap::new()) - .schema(schema) - .build() - } - - #[tokio::test] - async fn test_iceberg_write_exec() -> Result<()> { - // 1. Set up test environment - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_namespace".to_string()); - - // Create namespace - iceberg_catalog - .create_namespace(&namespace, HashMap::new()) - .await?; - - // Create schema - let schema = get_test_schema()?; - - // Create table - let table_name = "test_table"; - let table_location = temp_path(); - let creation = get_table_creation(table_location, table_name, schema); - let table = iceberg_catalog.create_table(&namespace, creation).await?; - - // 2. Create test data - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - ])); - - let id_array = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; - let name_array = Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])) as ArrayRef; - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_array, name_array]) - .map_err(|e| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to create record batch: {e}"), - ) - })?; - - // 3. Create mock input execution plan - let input_plan = Arc::new(MockExecutionPlan::new(arrow_schema.clone(), vec![ - batch.clone(), - ])); - - // 4. Create IcebergWriteExec - let write_exec = IcebergWriteExec::new(table.clone(), input_plan); - - // 5. Execute the plan - let task_ctx = Arc::new(TaskContext::default()); - let stream = write_exec.execute(0, task_ctx).map_err(|e| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to execute plan: {e}"), - ) - })?; - - // Collect the results - let mut results = vec![]; - let mut stream = stream; - while let Some(batch) = stream.next().await { - results.push(batch.map_err(|e| { - Error::new(ErrorKind::Unexpected, format!("Failed to get batch: {e}")) - })?); - } - - // 6. Verify the results - assert_eq!(results.len(), 1, "Expected one result batch"); - let result_batch = &results[0]; - - // Check schema - assert_eq!( - result_batch.schema().as_ref(), - &ArrowSchema::new(vec![Field::new(DATA_FILES_COL_NAME, DataType::Utf8, false)]) - ); - - // Check data - assert_eq!(result_batch.num_rows(), 1, "Expected one data file"); - - // Get the data file JSON - let data_file_json = result_batch - .column(0) - .as_any() - .downcast_ref::() - .expect("Expected StringArray") - .value(0); - - // Deserialize the data file JSON - let partition_type = table.metadata().default_partition_type(); - let spec_id = table.metadata().default_partition_spec_id(); - let schema = table.metadata().current_schema(); - - let data_file = - deserialize_data_file_from_json(data_file_json, spec_id, partition_type, schema) - .expect("Failed to deserialize data file JSON"); - - // Verify data file properties - assert_eq!( - data_file.record_count(), - 3, - "Expected 3 records in the data file" - ); - assert!( - data_file.file_size_in_bytes() > 0, - "File size should be greater than 0" - ); - assert_eq!( - data_file.file_format(), - DataFileFormat::Parquet, - "Expected Parquet file format" - ); - - // Verify column statistics - assert!( - data_file.column_sizes().get(&1).unwrap() > &0, - "Column 1 size should be greater than 0" - ); - assert!( - data_file.column_sizes().get(&2).unwrap() > &0, - "Column 2 size should be greater than 0" - ); - - assert_eq!( - *data_file.value_counts().get(&1).unwrap(), - 3, - "Expected 3 values for column 1" - ); - assert_eq!( - *data_file.value_counts().get(&2).unwrap(), - 3, - "Expected 3 values for column 2" - ); - - // Verify lower and upper bounds - assert!( - data_file.lower_bounds().contains_key(&1) || data_file.lower_bounds().contains_key(&2), - "Expected lower bounds to contain at least one column" - ); - assert!( - data_file.upper_bounds().contains_key(&1) || data_file.upper_bounds().contains_key(&2), - "Expected upper bounds to contain at least one column" - ); - - // Check that the file path exists - let file_path = data_file.file_path(); - assert!(!file_path.is_empty(), "File path should not be empty"); - - // 7. Verify the file exists - let file_io = table.file_io(); - assert!(file_io.exists(file_path).await?, "Data file should exist"); - - Ok(()) - } - - #[tokio::test] - async fn test_iceberg_write_exec_advertises_result_schema() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_namespace".to_string()); - iceberg_catalog - .create_namespace(&namespace, HashMap::new()) - .await?; - let creation = get_table_creation(temp_path(), "test_table", get_test_schema()?); - let table = iceberg_catalog.create_table(&namespace, creation).await?; - - let table_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - let input = Arc::new(MockExecutionPlan::new(table_schema.clone(), vec![])); - - let write_exec = IcebergWriteExec::new(table, input); - - assert_eq!( - write_exec.schema().as_ref(), - &ArrowSchema::new(vec![Field::new(DATA_FILES_COL_NAME, DataType::Utf8, false)]), - "IcebergWriteExec should advertise the data_files schema, not the table schema" - ); - - Ok(()) - } -} diff --git a/crates/integrations/datafusion/src/schema.rs b/crates/integrations/datafusion/src/schema.rs deleted file mode 100644 index 545863f8c6..0000000000 --- a/crates/integrations/datafusion/src/schema.rs +++ /dev/null @@ -1,447 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::Arc; - -use async_trait::async_trait; -use dashmap::DashMap; -use datafusion::catalog::SchemaProvider; -use datafusion::datasource::TableProvider; -use datafusion::error::{DataFusionError, Result as DFResult}; -use datafusion::execution::TaskContext; -use datafusion::prelude::SessionContext; -use futures::StreamExt; -use futures::future::try_join_all; -use iceberg::arrow::arrow_schema_to_schema_auto_assign_ids; -use iceberg::inspect::MetadataTableType; -use iceberg::spec::FormatVersion; -use iceberg::{Catalog, Error, ErrorKind, NamespaceIdent, Result, TableCreation, TableIdent}; - -use crate::table::IcebergTableProvider; -use crate::to_datafusion_error; - -/// Represents a [`SchemaProvider`] for the Iceberg [`Catalog`], managing -/// access to table providers within a specific namespace. -#[derive(Debug)] -pub(crate) struct IcebergSchemaProvider { - /// Reference to the Iceberg catalog - catalog: Arc, - /// The namespace this schema represents - namespace: NamespaceIdent, - /// A concurrent map where keys are table names - /// and values are dynamic references to objects implementing the - /// [`TableProvider`] trait. - /// Wrapped in Arc to allow sharing across async boundaries in register_table. - tables: Arc>>, -} - -impl IcebergSchemaProvider { - /// Asynchronously tries to construct a new [`IcebergSchemaProvider`] - /// using the given client to fetch and initialize table providers for - /// the provided namespace in the Iceberg [`Catalog`]. - /// - /// This method retrieves a list of table names - /// attempts to create a table provider for each table name, and - /// collects these providers into a `HashMap`. - pub(crate) async fn try_new( - client: Arc, - namespace: NamespaceIdent, - ) -> Result { - // TODO: - // Tables and providers should be cached based on table_name - // if we have a cache miss; we update our internal cache & check again - // As of right now; tables might become stale. - let table_names: Vec<_> = client - .list_tables(&namespace) - .await? - .iter() - .map(|tbl| tbl.name().to_string()) - .collect(); - - let providers = try_join_all( - table_names - .iter() - .map(|name| IcebergTableProvider::try_new(client.clone(), namespace.clone(), name)) - .collect::>(), - ) - .await?; - - let tables = Arc::new(DashMap::new()); - for (name, provider) in table_names.into_iter().zip(providers) { - tables.insert(name, Arc::new(provider)); - } - - Ok(IcebergSchemaProvider { - catalog: client, - namespace, - tables, - }) - } -} - -#[async_trait] -impl SchemaProvider for IcebergSchemaProvider { - fn table_names(&self) -> Vec { - self.tables - .iter() - .flat_map(|entry| { - let table_name = entry.key().clone(); - [table_name.clone()] - .into_iter() - .chain( - MetadataTableType::all_types().map(move |metadata_table_name| { - format!("{}${}", table_name, metadata_table_name.as_str()) - }), - ) - }) - .collect() - } - - fn table_exist(&self, name: &str) -> bool { - if let Some((table_name, metadata_table_name)) = name.split_once('$') { - self.tables.contains_key(table_name) - && MetadataTableType::try_from(metadata_table_name).is_ok() - } else { - self.tables.contains_key(name) - } - } - - async fn table(&self, name: &str) -> DFResult>> { - if let Some((table_name, metadata_table_name)) = name.split_once('$') { - let metadata_table_type = - MetadataTableType::try_from(metadata_table_name).map_err(DataFusionError::Plan)?; - if let Some(table) = self.tables.get(table_name) { - let metadata_table = table - .metadata_table(metadata_table_type) - .await - .map_err(to_datafusion_error)?; - return Ok(Some(Arc::new(metadata_table))); - } else { - return Ok(None); - } - } - - Ok(self - .tables - .get(name) - .map(|entry| entry.value().clone() as Arc)) - } - - fn register_table( - &self, - name: String, - table: Arc, - ) -> DFResult>> { - // Check if table already exists - if self.table_exist(name.as_str()) { - return Err(DataFusionError::Execution(format!( - "Table {name} already exists" - ))); - } - - // Convert DataFusion schema to Iceberg schema - // DataFusion schemas don't have field IDs, so we use the function that assigns them automatically - let df_schema = table.schema(); - let iceberg_schema = arrow_schema_to_schema_auto_assign_ids(df_schema.as_ref()) - .map_err(to_datafusion_error)?; - - // Use at least V2, and upgrade to V3 if the schema requires it (e.g. timestamp_ns / variant). - let format_version = iceberg_schema - .calc_min_compatible_format() - .max(FormatVersion::V2); - - // Create the table in the Iceberg catalog - let table_creation = TableCreation::builder() - .name(name.clone()) - .schema(iceberg_schema) - .format_version(format_version) - .build(); - - let catalog = self.catalog.clone(); - let namespace = self.namespace.clone(); - let tables = self.tables.clone(); - let name_clone = name.clone(); - - // Use tokio's spawn_blocking to handle the async work on a blocking thread pool - let result = tokio::task::spawn_blocking(move || { - // Create a new runtime handle to execute the async work - let rt = tokio::runtime::Handle::current(); - rt.block_on(async move { - // Verify the input table is empty - CREATE TABLE only accepts schema definition - ensure_table_is_empty(&table) - .await - .map_err(to_datafusion_error)?; - - catalog - .create_table(&namespace, table_creation) - .await - .map_err(to_datafusion_error)?; - - // Create a new table provider using the catalog reference - let table_provider = IcebergTableProvider::try_new( - catalog.clone(), - namespace.clone(), - name_clone.clone(), - ) - .await - .map_err(to_datafusion_error)?; - - // Store the new table provider - tables.insert(name_clone, Arc::new(table_provider)); - - Ok(None) - }) - }); - - // Block on the spawned task to get the result - // This is safe because spawn_blocking moves the blocking to a dedicated thread pool - futures::executor::block_on(result).map_err(|e| { - DataFusionError::Execution(format!("Failed to create Iceberg table: {e}")) - })? - } - - fn deregister_table(&self, name: &str) -> DFResult>> { - // Check if table exists - if !self.table_exist(name) { - return Ok(None); - } - - let catalog = self.catalog.clone(); - let namespace = self.namespace.clone(); - let tables = self.tables.clone(); - let table_name = name.to_string(); - - // Use tokio's spawn_blocking to handle the async work on a blocking thread pool - let result = tokio::task::spawn_blocking(move || { - let rt = tokio::runtime::Handle::current(); - rt.block_on(async move { - let table_ident = TableIdent::new(namespace, table_name.clone()); - - // Drop the table from the Iceberg catalog - catalog - .drop_table(&table_ident) - .await - .map_err(to_datafusion_error)?; - - // Remove from local cache and return the removed provider - let removed = tables - .remove(&table_name) - .map(|(_, table)| table as Arc); - - Ok(removed) - }) - }); - - futures::executor::block_on(result) - .map_err(|e| DataFusionError::Execution(format!("Failed to drop Iceberg table: {e}")))? - } -} - -/// Verifies that a table provider contains no data by scanning with LIMIT 1. -/// Returns an error if the table has any rows. -async fn ensure_table_is_empty(table: &Arc) -> Result<()> { - let session_ctx = SessionContext::new(); - let exec_plan = table - .scan(&session_ctx.state(), None, &[], Some(1)) - .await - .map_err(|e| Error::new(ErrorKind::Unexpected, format!("Failed to scan table: {e}")))?; - - let task_ctx = Arc::new(TaskContext::default()); - let stream = exec_plan.execute(0, task_ctx).map_err(|e| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to execute scan: {e}"), - ) - })?; - - let batches: Vec<_> = stream.collect().await; - let has_data = batches - .into_iter() - .filter_map(|r| r.ok()) - .any(|batch| batch.num_rows() > 0); - - if has_data { - return Err(Error::new( - ErrorKind::Unexpected, - "register_table does not support tables with data.", - )); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; - - use datafusion::arrow::array::{Int32Array, StringArray}; - use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; - use datafusion::arrow::record_batch::RecordBatch; - use datafusion::datasource::MemTable; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::{Catalog, CatalogBuilder, NamespaceIdent}; - use tempfile::TempDir; - - use super::*; - - async fn create_test_schema_provider() -> (IcebergSchemaProvider, TempDir) { - let temp_dir = TempDir::new().unwrap(); - let warehouse_path = temp_dir.path().to_str().unwrap().to_string(); - - let catalog = MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_path.clone())]), - ) - .await - .unwrap(); - - let namespace = NamespaceIdent::new("test_ns".to_string()); - catalog - .create_namespace(&namespace, HashMap::new()) - .await - .unwrap(); - - let provider = IcebergSchemaProvider::try_new(Arc::new(catalog), namespace) - .await - .unwrap(); - - (provider, temp_dir) - } - - #[tokio::test] - async fn test_register_table_with_data_fails() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Create a MemTable with data - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, true), - ])); - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])), - ]) - .unwrap(); - - let mem_table = MemTable::try_new(arrow_schema, vec![vec![batch]]).unwrap(); - - // Attempt to register the table with data - should fail - let result = schema_provider.register_table("test_table".to_string(), Arc::new(mem_table)); - - assert!(result.is_err()); - let err = result.unwrap_err(); - assert!( - err.to_string() - .contains("register_table does not support tables with data."), - "Expected error about tables with data, got: {err}", - ); - } - - #[tokio::test] - async fn test_register_empty_table_succeeds() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Create an empty MemTable (schema only, no data rows) - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, true), - ])); - - // Create an empty batch (0 rows) - MemTable requires at least one partition - let empty_batch = RecordBatch::new_empty(arrow_schema.clone()); - let mem_table = MemTable::try_new(arrow_schema, vec![vec![empty_batch]]).unwrap(); - - // Attempt to register the empty table - should succeed - let result = schema_provider.register_table("empty_table".to_string(), Arc::new(mem_table)); - - assert!(result.is_ok(), "Expected success, got: {result:?}"); - - // Verify the table was registered - assert!(schema_provider.table_exist("empty_table")); - } - - #[tokio::test] - async fn test_register_duplicate_table_fails() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Create empty MemTables - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - "id", - DataType::Int32, - false, - )])); - - let empty_batch1 = RecordBatch::new_empty(arrow_schema.clone()); - let empty_batch2 = RecordBatch::new_empty(arrow_schema.clone()); - let mem_table1 = MemTable::try_new(arrow_schema.clone(), vec![vec![empty_batch1]]).unwrap(); - let mem_table2 = MemTable::try_new(arrow_schema, vec![vec![empty_batch2]]).unwrap(); - - // Register first table - should succeed - let result1 = schema_provider.register_table("dup_table".to_string(), Arc::new(mem_table1)); - assert!(result1.is_ok()); - - // Register second table with same name - should fail - let result2 = schema_provider.register_table("dup_table".to_string(), Arc::new(mem_table2)); - assert!(result2.is_err()); - let err = result2.unwrap_err(); - assert!( - err.to_string().contains("already exists"), - "Expected error about table already existing, got: {err}", - ); - } - - #[tokio::test] - async fn test_deregister_table_succeeds() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Create and register an empty table - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - "id", - DataType::Int32, - false, - )])); - - let empty_batch = RecordBatch::new_empty(arrow_schema.clone()); - let mem_table = MemTable::try_new(arrow_schema, vec![vec![empty_batch]]).unwrap(); - - // Register the table - let result = schema_provider.register_table("drop_me".to_string(), Arc::new(mem_table)); - assert!(result.is_ok()); - assert!(schema_provider.table_exist("drop_me")); - - // Deregister the table - let result = schema_provider.deregister_table("drop_me"); - assert!(result.is_ok()); - assert!(result.unwrap().is_some()); - - // Verify the table no longer exists - assert!(!schema_provider.table_exist("drop_me")); - } - - #[tokio::test] - async fn test_deregister_nonexistent_table_returns_none() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Attempt to deregister a table that doesn't exist - let result = schema_provider.deregister_table("nonexistent"); - assert!(result.is_ok()); - assert!(result.unwrap().is_none()); - } -} diff --git a/crates/integrations/datafusion/src/table/metadata_table.rs b/crates/integrations/datafusion/src/table/metadata_table.rs deleted file mode 100644 index 56d0c10e35..0000000000 --- a/crates/integrations/datafusion/src/table/metadata_table.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::catalog::Session; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use futures::TryStreamExt; -use futures::stream::BoxStream; -use iceberg::arrow::schema_to_arrow_schema; -use iceberg::inspect::MetadataTableType; -use iceberg::table::Table; - -use crate::physical_plan::metadata_scan::IcebergMetadataScan; -use crate::to_datafusion_error; - -/// Represents a [`TableProvider`] for the Iceberg [`iceberg::Catalog`], -/// managing access to a [`iceberg::inspect::MetadataTable`]. -#[derive(Debug, Clone)] -pub struct IcebergMetadataTableProvider { - pub(crate) table: Table, - pub(crate) r#type: MetadataTableType, -} - -#[async_trait] -impl TableProvider for IcebergMetadataTableProvider { - fn schema(&self) -> ArrowSchemaRef { - let metadata_table = self.table.inspect(); - let schema = match self.r#type { - MetadataTableType::Snapshots => metadata_table.snapshots().schema(), - MetadataTableType::Manifests => metadata_table.manifests().schema(), - MetadataTableType::History => metadata_table.history().schema(), - }; - schema_to_arrow_schema(&schema).unwrap().into() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - _projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - Ok(Arc::new(IcebergMetadataScan::new(self.clone()))) - } -} - -impl IcebergMetadataTableProvider { - pub async fn scan(self) -> DFResult>> { - let metadata_table = self.table.inspect(); - let stream = match self.r#type { - MetadataTableType::Snapshots => metadata_table.snapshots().scan().await, - MetadataTableType::Manifests => metadata_table.manifests().scan().await, - MetadataTableType::History => metadata_table.history().scan().await, - } - .map_err(to_datafusion_error)?; - let stream = stream.map_err(to_datafusion_error); - Ok(Box::pin(stream)) - } -} diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs deleted file mode 100644 index 9de7bcb9c2..0000000000 --- a/crates/integrations/datafusion/src/table/mod.rs +++ /dev/null @@ -1,893 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Iceberg table providers for DataFusion. -//! -//! This module provides two table provider implementations: -//! -//! - [`IcebergTableProvider`]: Catalog-backed provider with automatic metadata refresh. -//! Use for write operations and when you need to see the latest table state. -//! -//! - [`IcebergStaticTableProvider`]: Static provider for read-only access to a specific -//! table snapshot. Use for consistent analytical queries or time-travel scenarios. - -pub mod metadata_table; -pub mod table_provider_factory; - -use std::num::NonZeroUsize; -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::catalog::Session; -use datafusion::common::DataFusionError; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::dml::InsertOp; -use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; -use datafusion::physical_plan::ExecutionPlan; -use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; -use iceberg::arrow::schema_to_arrow_schema; -use iceberg::inspect::MetadataTableType; -use iceberg::spec::TableProperties; -use iceberg::table::Table; -use iceberg::{Catalog, Error, ErrorKind, NamespaceIdent, Result, TableIdent}; -use metadata_table::IcebergMetadataTableProvider; - -use crate::error::to_datafusion_error; -use crate::physical_plan::commit::IcebergCommitExec; -use crate::physical_plan::project::project_with_partition; -use crate::physical_plan::repartition::repartition; -use crate::physical_plan::scan::IcebergTableScan; -use crate::physical_plan::sort::sort_by_partition; -use crate::physical_plan::write::IcebergWriteExec; - -/// Catalog-backed table provider with automatic metadata refresh. -/// -/// This provider loads fresh table metadata from the catalog on every scan and write -/// operation, ensuring you always see the latest table state. Use this when you need -/// write operations or want to see the most up-to-date data. -/// -/// For read-only access to a specific snapshot without catalog overhead, use -/// [`IcebergStaticTableProvider`] instead. -#[derive(Debug, Clone)] -pub struct IcebergTableProvider { - /// The catalog that manages this table - catalog: Arc, - /// The table identifier (namespace + name) - table_ident: TableIdent, - /// A reference-counted arrow `Schema` (cached at construction) - schema: ArrowSchemaRef, -} - -impl IcebergTableProvider { - /// Creates a new catalog-backed table provider. - /// - /// Loads the table once to get the initial schema, then stores the catalog - /// reference for future metadata refreshes on each operation. - pub(crate) async fn try_new( - catalog: Arc, - namespace: NamespaceIdent, - name: impl Into, - ) -> Result { - let table_ident = TableIdent::new(namespace, name.into()); - - // Load table once to get initial schema - let table = catalog.load_table(&table_ident).await?; - let schema = Arc::new(schema_to_arrow_schema(table.metadata().current_schema())?); - - Ok(IcebergTableProvider { - catalog, - table_ident, - schema, - }) - } - - pub(crate) async fn metadata_table( - &self, - r#type: MetadataTableType, - ) -> Result { - // Load fresh table metadata for metadata table access - let table = self.catalog.load_table(&self.table_ident).await?; - Ok(IcebergMetadataTableProvider { table, r#type }) - } -} - -#[async_trait] -impl TableProvider for IcebergTableProvider { - fn schema(&self) -> ArrowSchemaRef { - self.schema.clone() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> DFResult> { - // Load fresh table metadata from catalog - let table = self - .catalog - .load_table(&self.table_ident) - .await - .map_err(to_datafusion_error)?; - - // Create scan with fresh metadata (always use current snapshot) - Ok(Arc::new(IcebergTableScan::new( - table, - None, // Always use current snapshot for catalog-backed provider - self.schema.clone(), - projection, - filters, - limit, - ))) - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> DFResult> { - // Push down all filters, as a single source of truth, the scanner will drop the filters which couldn't be push down - Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) - } - - async fn insert_into( - &self, - state: &dyn Session, - input: Arc, - _insert_op: InsertOp, - ) -> DFResult> { - if _insert_op != InsertOp::Append { - return Err(DataFusionError::NotImplemented(format!( - "IcebergTableProvider supports only append inserts, got {_insert_op}" - ))); - } - - // Load fresh table metadata from catalog - let table = self - .catalog - .load_table(&self.table_ident) - .await - .map_err(to_datafusion_error)?; - - let partition_spec = table.metadata().default_partition_spec(); - - // Step 1: Project partition values for partitioned tables - let plan_with_partition = if !partition_spec.is_unpartitioned() { - project_with_partition(input, &table)? - } else { - input - }; - - // Step 2: Repartition for parallel processing - let target_partitions = - NonZeroUsize::new(state.config().target_partitions()).ok_or_else(|| { - DataFusionError::Configuration( - "target_partitions must be greater than 0".to_string(), - ) - })?; - - let repartitioned_plan = - repartition(plan_with_partition, table.metadata_ref(), target_partitions)?; - - // Apply sort node when it's not fanout mode - let fanout_enabled = table - .metadata() - .properties() - .get(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED) - .map(|value| { - value - .parse::() - .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid value for {}, expected 'true' or 'false'", - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED - ), - ) - .with_source(e) - }) - .map_err(to_datafusion_error) - }) - .transpose()? - .unwrap_or(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT); - - let write_input = if fanout_enabled { - repartitioned_plan - } else { - sort_by_partition(repartitioned_plan)? - }; - - let write_plan = Arc::new(IcebergWriteExec::new(table.clone(), write_input)); - - // Merge the outputs of write_plan into one so we can commit all files together - let coalesce_partitions = Arc::new(CoalescePartitionsExec::new(write_plan)); - - Ok(Arc::new(IcebergCommitExec::new( - table, - self.catalog.clone(), - coalesce_partitions, - self.schema.clone(), - ))) - } -} - -/// Static table provider for read-only snapshot access. -/// -/// This provider holds a cached table instance and does not refresh metadata or support -/// write operations. Use this for consistent analytical queries, time-travel scenarios, -/// or when you want to avoid catalog overhead. -/// -/// For catalog-backed tables with write support and automatic refresh, use -/// [`IcebergTableProvider`] instead. -#[derive(Debug, Clone)] -pub struct IcebergStaticTableProvider { - /// The static table instance (never refreshed) - table: Table, - /// Optional snapshot ID for this static view - snapshot_id: Option, - /// A reference-counted arrow `Schema` - schema: ArrowSchemaRef, -} - -impl IcebergStaticTableProvider { - /// Creates a static provider from a table instance. - /// - /// Uses the table's current snapshot for all queries. Does not support write operations. - pub async fn try_new_from_table(table: Table) -> Result { - let schema = Arc::new(schema_to_arrow_schema(table.metadata().current_schema())?); - Ok(IcebergStaticTableProvider { - table, - snapshot_id: None, - schema, - }) - } - - /// Creates a static provider for a specific table snapshot. - /// - /// Queries the specified snapshot for all operations. Useful for time-travel queries. - /// Does not support write operations. - pub async fn try_new_from_table_snapshot(table: Table, snapshot_id: i64) -> Result { - let snapshot = table - .metadata() - .snapshot_by_id(snapshot_id) - .ok_or_else(|| { - Error::new( - ErrorKind::Unexpected, - format!( - "snapshot id {snapshot_id} not found in table {}", - table.identifier().name() - ), - ) - })?; - let table_schema = snapshot.schema(table.metadata())?; - let schema = Arc::new(schema_to_arrow_schema(&table_schema)?); - Ok(IcebergStaticTableProvider { - table, - snapshot_id: Some(snapshot_id), - schema, - }) - } -} - -#[async_trait] -impl TableProvider for IcebergStaticTableProvider { - fn schema(&self) -> ArrowSchemaRef { - self.schema.clone() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> DFResult> { - // Use cached table (no refresh) - Ok(Arc::new(IcebergTableScan::new( - self.table.clone(), - self.snapshot_id, - self.schema.clone(), - projection, - filters, - limit, - ))) - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> DFResult> { - // Push down all filters, as a single source of truth, the scanner will drop the filters which couldn't be push down - Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) - } - - async fn insert_into( - &self, - _state: &dyn Session, - _input: Arc, - _insert_op: InsertOp, - ) -> DFResult> { - Err(to_datafusion_error(Error::new( - ErrorKind::FeatureUnsupported, - "Write operations are not supported on IcebergStaticTableProvider. \ - Use IcebergTableProvider with a catalog for write support." - .to_string(), - ))) - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; - - use datafusion::common::Column; - use datafusion::physical_plan::ExecutionPlan; - use datafusion::prelude::SessionContext; - use iceberg::io::FileIO; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; - use iceberg::table::{StaticTable, Table}; - use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; - use tempfile::TempDir; - - use super::*; - - async fn get_test_table_from_metadata_file() -> Table { - let metadata_file_name = "TableMetadataV2Valid.json"; - let metadata_file_path = format!( - "{}/tests/test_data/{}", - env!("CARGO_MANIFEST_DIR"), - metadata_file_name - ); - let file_io = FileIO::new_with_fs(); - let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap(); - let static_table = - StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io) - .await - .unwrap(); - static_table.into_table() - } - - async fn get_test_catalog_and_table() -> (Arc, NamespaceIdent, String, TempDir) { - let temp_dir = TempDir::new().unwrap(); - let warehouse_path = temp_dir.path().to_str().unwrap().to_string(); - - let catalog = MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_path.clone())]), - ) - .await - .unwrap(); - - let namespace = NamespaceIdent::new("test_ns".to_string()); - catalog - .create_namespace(&namespace, HashMap::new()) - .await - .unwrap(); - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let table_creation = TableCreation::builder() - .name("test_table".to_string()) - .location(format!("{warehouse_path}/test_table")) - .schema(schema) - .properties(HashMap::new()) - .build(); - - catalog - .create_table(&namespace, table_creation) - .await - .unwrap(); - - ( - Arc::new(catalog), - namespace, - "test_table".to_string(), - temp_dir, - ) - } - - // Tests for IcebergStaticTableProvider - - #[tokio::test] - async fn test_static_provider_from_table() { - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - let df = ctx.sql("SELECT * FROM mytable").await.unwrap(); - let df_schema = df.schema(); - let df_columns = df_schema.fields(); - assert_eq!(df_columns.len(), 3); - let x_column = df_columns.first().unwrap(); - let column_data = format!( - "{:?}:{:?}", - x_column.name(), - x_column.data_type().to_string() - ); - assert_eq!(column_data, "\"x\":\"Int64\""); - let has_column = df_schema.has_column(&Column::from_name("z")); - assert!(has_column); - } - - #[tokio::test] - async fn test_static_provider_from_snapshot() { - let table = get_test_table_from_metadata_file().await; - let snapshot_id = table.metadata().snapshots().next().unwrap().snapshot_id(); - let table_provider = - IcebergStaticTableProvider::try_new_from_table_snapshot(table.clone(), snapshot_id) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - let df = ctx.sql("SELECT * FROM mytable").await.unwrap(); - let df_schema = df.schema(); - let df_columns = df_schema.fields(); - assert_eq!(df_columns.len(), 3); - let x_column = df_columns.first().unwrap(); - let column_data = format!( - "{:?}:{:?}", - x_column.name(), - x_column.data_type().to_string() - ); - assert_eq!(column_data, "\"x\":\"Int64\""); - let has_column = df_schema.has_column(&Column::from_name("z")); - assert!(has_column); - } - - #[tokio::test] - async fn test_static_provider_rejects_writes() { - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - - // Attempt to insert into the static provider should fail - let result = ctx.sql("INSERT INTO mytable VALUES (1, 2, 3)").await; - - // The error should occur during planning or execution - // We expect an error indicating write operations are not supported - assert!( - result.is_err() || { - let df = result.unwrap(); - df.collect().await.is_err() - } - ); - } - - #[tokio::test] - async fn test_static_provider_scan() { - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - - // Test that scan operations work correctly - let df = ctx.sql("SELECT count(*) FROM mytable").await.unwrap(); - let physical_plan = df.create_physical_plan().await; - assert!(physical_plan.is_ok()); - } - - // Tests for IcebergTableProvider - - #[tokio::test] - async fn test_catalog_backed_provider_creation() { - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - // Test creating a catalog-backed provider - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - // Verify the schema is loaded correctly - let schema = provider.schema(); - assert_eq!(schema.fields().len(), 2); - assert_eq!(schema.field(0).name(), "id"); - assert_eq!(schema.field(1).name(), "name"); - } - - #[tokio::test] - async fn test_catalog_backed_provider_scan() { - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - ctx.register_table("test_table", Arc::new(provider)) - .unwrap(); - - // Test that scan operations work correctly - let df = ctx.sql("SELECT * FROM test_table").await.unwrap(); - - // Verify the schema in the query result - let df_schema = df.schema(); - assert_eq!(df_schema.fields().len(), 2); - assert_eq!(df_schema.field(0).name(), "id"); - assert_eq!(df_schema.field(1).name(), "name"); - - let physical_plan = df.create_physical_plan().await; - assert!(physical_plan.is_ok()); - } - - #[tokio::test] - async fn test_catalog_backed_provider_insert() { - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - ctx.register_table("test_table", Arc::new(provider)) - .unwrap(); - - // Test that insert operations work correctly - let result = ctx.sql("INSERT INTO test_table VALUES (1, 'test')").await; - - // Insert should succeed (or at least not fail during planning) - assert!(result.is_ok()); - - // Try to execute the insert plan - let df = result.unwrap(); - let execution_result = df.collect().await; - - // The execution should succeed - assert!(execution_result.is_ok()); - } - - #[tokio::test] - async fn test_physical_input_schema_consistent_with_logical_input_schema() { - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - ctx.register_table("test_table", Arc::new(provider)) - .unwrap(); - - // Create a query plan - let df = ctx.sql("SELECT id, name FROM test_table").await.unwrap(); - - // Get logical schema before consuming df - let logical_schema = df.schema().clone(); - - // Get physical plan (this consumes df) - let physical_plan = df.create_physical_plan().await.unwrap(); - let physical_schema = physical_plan.schema(); - - // Verify that logical and physical schemas are consistent - assert_eq!( - logical_schema.fields().len(), - physical_schema.fields().len() - ); - - for (logical_field, physical_field) in logical_schema - .fields() - .iter() - .zip(physical_schema.fields().iter()) - { - assert_eq!(logical_field.name(), physical_field.name()); - assert_eq!(logical_field.data_type(), physical_field.data_type()); - } - } - - async fn get_partitioned_test_catalog_and_table( - fanout_enabled: Option, - ) -> (Arc, NamespaceIdent, String, TempDir) { - use iceberg::spec::{Transform, UnboundPartitionSpec}; - - let temp_dir = TempDir::new().unwrap(); - let warehouse_path = temp_dir.path().to_str().unwrap().to_string(); - - let catalog = MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_path.clone())]), - ) - .await - .unwrap(); - - let namespace = NamespaceIdent::new("test_ns".to_string()); - catalog - .create_namespace(&namespace, HashMap::new()) - .await - .unwrap(); - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = UnboundPartitionSpec::builder() - .with_spec_id(0) - .add_partition_field(2, "category", Transform::Identity) - .unwrap() - .build(); - - let mut properties = HashMap::new(); - if let Some(enabled) = fanout_enabled { - properties.insert( - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED.to_string(), - enabled.to_string(), - ); - } - - let table_creation = TableCreation::builder() - .name("partitioned_table".to_string()) - .location(format!("{warehouse_path}/partitioned_table")) - .schema(schema) - .partition_spec(partition_spec) - .properties(properties) - .build(); - - catalog - .create_table(&namespace, table_creation) - .await - .unwrap(); - - ( - Arc::new(catalog), - namespace, - "partitioned_table".to_string(), - temp_dir, - ) - } - - /// Helper to check if a plan contains a SortExec node - fn plan_contains_sort(plan: &Arc) -> bool { - if plan.name() == "SortExec" { - return true; - } - for child in plan.children() { - if plan_contains_sort(child) { - return true; - } - } - false - } - - #[tokio::test] - async fn test_catalog_backed_provider_rejects_non_append_op() { - use datafusion::physical_plan::empty::EmptyExec; - - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - let provider = IcebergTableProvider::try_new(catalog, namespace, table_name) - .await - .unwrap(); - let ctx = SessionContext::new(); - - for (insert_op, expected_message) in [ - ( - InsertOp::Overwrite, - "IcebergTableProvider supports only append inserts, got Insert Overwrite", - ), - ( - InsertOp::Replace, - "IcebergTableProvider supports only append inserts, got Replace Into", - ), - ] { - let input = Arc::new(EmptyExec::new(provider.schema())) as Arc; - let error = provider - .insert_into(&ctx.state(), input, insert_op) - .await - .expect_err("non-append inserts should be rejected"); - - assert!( - matches!( - error, - DataFusionError::NotImplemented(ref message) if message == expected_message - ), - "unexpected error: {error}" - ); - } - } - - #[tokio::test] - async fn test_insert_plan_fanout_enabled_no_sort() { - use datafusion::datasource::TableProvider; - use datafusion::logical_expr::dml::InsertOp; - use datafusion::physical_plan::empty::EmptyExec; - - // When fanout is enabled (default), no sort node should be added - let (catalog, namespace, table_name, _temp_dir) = - get_partitioned_test_catalog_and_table(Some(true)).await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let input_schema = provider.schema(); - let input = Arc::new(EmptyExec::new(input_schema)) as Arc; - - let state = ctx.state(); - let insert_plan = provider - .insert_into(&state, input, InsertOp::Append) - .await - .unwrap(); - - // With fanout enabled, there should be no SortExec in the plan - assert!( - !plan_contains_sort(&insert_plan), - "Plan should NOT contain SortExec when fanout is enabled" - ); - } - - #[tokio::test] - async fn test_insert_plan_fanout_disabled_has_sort() { - use datafusion::datasource::TableProvider; - use datafusion::logical_expr::dml::InsertOp; - use datafusion::physical_plan::empty::EmptyExec; - - // When fanout is disabled, a sort node should be added - let (catalog, namespace, table_name, _temp_dir) = - get_partitioned_test_catalog_and_table(Some(false)).await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let input_schema = provider.schema(); - let input = Arc::new(EmptyExec::new(input_schema)) as Arc; - - let state = ctx.state(); - let insert_plan = provider - .insert_into(&state, input, InsertOp::Append) - .await - .unwrap(); - - // With fanout disabled, there should be a SortExec in the plan - assert!( - plan_contains_sort(&insert_plan), - "Plan should contain SortExec when fanout is disabled" - ); - } - - #[tokio::test] - async fn test_limit_pushdown_static_provider() { - use datafusion::datasource::TableProvider; - - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let state = ctx.state(); - - // Test scan with limit - let scan_plan = table_provider - .scan(&state, None, &[], Some(10)) - .await - .unwrap(); - - // Verify that the scan plan is an IcebergTableScan - let iceberg_scan = scan_plan - .downcast_ref::() - .expect("Expected IcebergTableScan"); - - // Verify the limit is set - assert_eq!( - iceberg_scan.limit(), - Some(10), - "Limit should be set to 10 in the scan plan" - ); - } - - #[tokio::test] - async fn test_limit_pushdown_catalog_backed_provider() { - use datafusion::datasource::TableProvider; - - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let state = ctx.state(); - - // Test scan with limit - let scan_plan = provider.scan(&state, None, &[], Some(5)).await.unwrap(); - - // Verify that the scan plan is an IcebergTableScan - let iceberg_scan = scan_plan - .downcast_ref::() - .expect("Expected IcebergTableScan"); - - // Verify the limit is set - assert_eq!( - iceberg_scan.limit(), - Some(5), - "Limit should be set to 5 in the scan plan" - ); - } - - #[tokio::test] - async fn test_no_limit_pushdown() { - use datafusion::datasource::TableProvider; - - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let state = ctx.state(); - - // Test scan without limit - let scan_plan = table_provider.scan(&state, None, &[], None).await.unwrap(); - - // Verify that the scan plan is an IcebergTableScan - let iceberg_scan = scan_plan - .downcast_ref::() - .expect("Expected IcebergTableScan"); - - // Verify the limit is None - assert_eq!( - iceberg_scan.limit(), - None, - "Limit should be None when not specified" - ); - } -} diff --git a/crates/integrations/datafusion/src/table/table_provider_factory.rs b/crates/integrations/datafusion/src/table/table_provider_factory.rs deleted file mode 100644 index a3b293f33a..0000000000 --- a/crates/integrations/datafusion/src/table/table_provider_factory.rs +++ /dev/null @@ -1,331 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::borrow::Cow; -use std::collections::HashMap; -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::catalog::{Session, TableProvider, TableProviderFactory}; -use datafusion::common::TableReference; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::CreateExternalTable; -use iceberg::io::{FileIOBuilder, LocalFsStorageFactory, StorageFactory}; -use iceberg::table::StaticTable; -use iceberg::{Error, ErrorKind, Result, TableIdent}; - -use super::IcebergStaticTableProvider; -use crate::to_datafusion_error; - -/// A factory that implements DataFusion's `TableProviderFactory` to create `IcebergTableProvider` instances. -/// -/// # Example -/// -/// The following example demonstrates how to create an Iceberg external table using SQL in -/// a DataFusion session with `IcebergTableProviderFactory`: -/// -/// ``` -/// use std::sync::Arc; -/// -/// use datafusion::common::TableReference; -/// use datafusion::execution::session_state::SessionStateBuilder; -/// use datafusion::prelude::*; -/// use iceberg_datafusion::IcebergTableProviderFactory; -/// -/// #[tokio::main] -/// async fn main() { -/// // Create a new session context -/// let mut state = SessionStateBuilder::new().with_default_features().build(); -/// -/// // Register the IcebergTableProviderFactory in the session -/// state.table_factories_mut().insert( -/// "ICEBERG".to_string(), -/// Arc::new(IcebergTableProviderFactory::new()), -/// ); -/// -/// let ctx = SessionContext::new_with_state(state); -/// -/// // Define the table reference and the location of the Iceberg metadata file -/// let table_ref = TableReference::bare("my_iceberg_table"); -/// // /path/to/iceberg/metadata -/// let metadata_file_path = format!( -/// "{}/testdata/table_metadata/{}", -/// env!("CARGO_MANIFEST_DIR"), -/// "TableMetadataV2.json" -/// ); -/// -/// // SQL command to create the Iceberg external table -/// let sql = format!( -/// "CREATE EXTERNAL TABLE {} STORED AS ICEBERG LOCATION '{}'", -/// table_ref, metadata_file_path -/// ); -/// -/// // Execute the SQL to create the external table -/// ctx.sql(&sql).await.expect("Failed to create table"); -/// -/// // Verify the table was created by retrieving the table provider -/// let table_provider = ctx -/// .table_provider(table_ref) -/// .await -/// .expect("Table not found"); -/// -/// println!("Iceberg external table created successfully."); -/// } -/// ``` -/// -/// # Note -/// This factory is designed to work with the DataFusion query engine, -/// specifically for handling Iceberg tables in external table commands. -/// Currently, this implementation supports only reading Iceberg tables, with -/// the creation of new tables not yet available. -/// -/// # Errors -/// An error will be returned if any unsupported feature, such as partition columns, -/// order expressions, constraints, or column defaults, is detected in the table creation command. -#[derive(Debug, Default)] -pub struct IcebergTableProviderFactory { - storage_factory: Option>, -} - -impl IcebergTableProviderFactory { - pub fn new() -> Self { - Self { - storage_factory: None, - } - } - - /// Create a new factory with a custom storage factory for creating FileIO instances. - pub fn new_with_storage_factory(storage_factory: Arc) -> Self { - Self { - storage_factory: Some(storage_factory), - } - } -} - -#[async_trait] -impl TableProviderFactory for IcebergTableProviderFactory { - async fn create( - &self, - _state: &dyn Session, - cmd: &CreateExternalTable, - ) -> DFResult> { - let metadata_file_path = check_cmd(cmd).map_err(to_datafusion_error)?; - - let table_name = &cmd.name; - let options = &cmd.options; - - let table_name_with_ns = complement_namespace_if_necessary(table_name); - - let storage_factory = self - .storage_factory - .clone() - .unwrap_or_else(|| Arc::new(LocalFsStorageFactory)); - - let table = create_static_table( - table_name_with_ns, - metadata_file_path, - options, - storage_factory, - ) - .await - .map_err(to_datafusion_error)? - .into_table(); - - let provider = IcebergStaticTableProvider::try_new_from_table(table) - .await - .map_err(to_datafusion_error)?; - - Ok(Arc::new(provider)) - } -} - -fn check_cmd(cmd: &CreateExternalTable) -> Result<&str> { - let CreateExternalTable { - schema, - table_partition_cols, - order_exprs, - constraints, - column_defaults, - .. - } = cmd; - - // Check if any of the fields violate the constraints in a single condition - let is_invalid = !schema.fields().is_empty() - || !table_partition_cols.is_empty() - || !order_exprs.is_empty() - || !constraints.is_empty() - || !column_defaults.is_empty(); - - if is_invalid { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Currently we only support reading existing icebergs tables in external table command. To create new table, please use catalog provider.", - )); - } - - match cmd.locations.as_slice() { - [location] => Ok(location), - _ => Err(Error::new( - ErrorKind::FeatureUnsupported, - "Iceberg external tables require exactly one metadata location.", - )), - } -} - -/// Complements the namespace of a table name if necessary. -/// -/// # Note -/// If the table name is a bare name, it will be complemented with the 'default' namespace. -/// Otherwise, it will be returned as is. Because Iceberg tables are always namespaced, but DataFusion -/// external table commands maybe not include the namespace, this function ensures that the namespace is always present. -/// -/// # See also -/// - [`iceberg::NamespaceIdent`] -/// - [`datafusion::sql::planner::SqlToRel::external_table_to_plan`] -fn complement_namespace_if_necessary(table_name: &TableReference) -> Cow<'_, TableReference> { - match table_name { - TableReference::Bare { table } => { - Cow::Owned(TableReference::partial("default", table.as_ref())) - } - other => Cow::Borrowed(other), - } -} - -async fn create_static_table( - table_name: Cow<'_, TableReference>, - metadata_file_path: &str, - props: &HashMap, - storage_factory: Arc, -) -> Result { - let table_ident = TableIdent::from_strs(table_name.to_vec())?; - let file_io = FileIOBuilder::new(storage_factory) - .with_props(props) - .build(); - StaticTable::from_metadata_file(metadata_file_path, table_ident, file_io).await -} - -#[cfg(test)] -mod tests { - - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::catalog::TableProviderFactory; - use datafusion::common::{Constraints, DFSchema, TableReference}; - use datafusion::execution::session_state::SessionStateBuilder; - use datafusion::logical_expr::CreateExternalTable; - use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; - use datafusion::prelude::SessionContext; - - use super::*; - - fn table_metadata_v2_schema() -> Schema { - Schema::new(vec![ - Field::new("x", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("y", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("z", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - ]) - } - - fn table_metadata_location() -> String { - format!( - "{}/testdata/table_metadata/{}", - env!("CARGO_MANIFEST_DIR"), - "TableMetadataV2.json" - ) - } - - fn create_external_table_cmd() -> CreateExternalTable { - let metadata_file_path = table_metadata_location(); - - CreateExternalTable { - name: TableReference::partial("static_ns", "static_table"), - locations: vec![metadata_file_path], - schema: Arc::new(DFSchema::empty()), - file_type: "iceberg".to_string(), - options: Default::default(), - table_partition_cols: Default::default(), - order_exprs: Default::default(), - constraints: Constraints::default(), - column_defaults: Default::default(), - if_not_exists: Default::default(), - or_replace: false, - temporary: false, - definition: Default::default(), - unbounded: Default::default(), - } - } - - #[tokio::test] - async fn test_schema_of_created_table() { - let factory = IcebergTableProviderFactory::new(); - - let state = SessionStateBuilder::new().build(); - let cmd = create_external_table_cmd(); - - let table_provider = factory - .create(&state, &cmd) - .await - .expect("create table failed"); - - let expected_schema = table_metadata_v2_schema(); - let actual_schema = table_provider.schema(); - - assert_eq!(actual_schema.as_ref(), &expected_schema); - } - - #[tokio::test] - async fn test_schema_of_created_external_table_sql() { - let mut state = SessionStateBuilder::new().with_default_features().build(); - state.table_factories_mut().insert( - "ICEBERG".to_string(), - Arc::new(IcebergTableProviderFactory::new()), - ); - let ctx = SessionContext::new_with_state(state); - - // All external tables in DataFusion use bare names. - // See https://github.com/apache/datafusion/blob/main/datafusion/sql/src/statement.rs#L1038-#L1039 - let table_ref = TableReference::bare("static_table"); - - // Create the external table - let sql = format!( - "CREATE EXTERNAL TABLE {} STORED AS ICEBERG LOCATION '{}'", - table_ref, - table_metadata_location() - ); - let _df = ctx.sql(&sql).await.expect("create table failed"); - - // Get the created external table - let table_provider = ctx - .table_provider(table_ref) - .await - .expect("table not found"); - - // Check the schema of the created table - let expected_schema = table_metadata_v2_schema(); - let actual_schema = table_provider.schema(); - - assert_eq!(actual_schema.as_ref(), &expected_schema); - } -} diff --git a/crates/integrations/datafusion/src/task_writer.rs b/crates/integrations/datafusion/src/task_writer.rs deleted file mode 100644 index cf44f60220..0000000000 --- a/crates/integrations/datafusion/src/task_writer.rs +++ /dev/null @@ -1,517 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! TaskWriter for DataFusion integration. -//! -//! This module provides a high-level writer that handles partitioning and routing -//! of RecordBatch data to Iceberg tables. - -use datafusion::arrow::array::RecordBatch; -use iceberg::Result; -use iceberg::arrow::RecordBatchPartitionSplitter; -use iceberg::spec::{DataFile, PartitionSpecRef, SchemaRef}; -use iceberg::writer::IcebergWriterBuilder; -use iceberg::writer::partitioning::PartitioningWriter; -use iceberg::writer::partitioning::clustered_writer::ClusteredWriter; -use iceberg::writer::partitioning::fanout_writer::FanoutWriter; -use iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter; - -/// High-level writer for DataFusion that handles partitioning and routing of RecordBatch data. -/// -/// TaskWriter coordinates writing data to Iceberg tables by: -/// - Selecting the appropriate partitioning strategy (unpartitioned, fanout, or clustered) -/// - Initializing the partition splitter in the constructor for partitioned tables -/// - Routing data to the underlying writer -/// - Collecting all written data files -/// -/// # Type Parameters -/// -/// * `B` - The IcebergWriterBuilder type used to create underlying writers -/// -/// # Example -/// -/// ```rust,ignore -/// use iceberg::spec::{PartitionSpec, Schema}; -/// use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -/// use iceberg_datafusion::writer::task_writer::TaskWriter; -/// -/// // Create a TaskWriter for an unpartitioned table -/// let task_writer = TaskWriter::new( -/// data_file_writer_builder, -/// false, // fanout_enabled -/// schema, -/// partition_spec, -/// ); -/// -/// // Write data -/// task_writer.write(record_batch).await?; -/// -/// // Close and get data files -/// let data_files = task_writer.close().await?; -/// ``` -pub(crate) struct TaskWriter { - /// The underlying writer (UnpartitionedWriter, FanoutWriter, or ClusteredWriter) - writer: SupportedWriter, - /// Partition splitter for partitioned tables (initialized in constructor) - partition_splitter: Option, -} - -/// Internal enum to hold the different writer types. -/// -/// This enum allows TaskWriter to work with different partitioning strategies -/// while maintaining a unified interface. -enum SupportedWriter { - /// Writer for unpartitioned tables - Unpartitioned(UnpartitionedWriter), - /// Writer for partitioned tables with unsorted data (maintains multiple active writers) - Fanout(FanoutWriter), - /// Writer for partitioned tables with sorted data (maintains single active writer) - Clustered(ClusteredWriter), -} - -impl TaskWriter { - /// Create a new TaskWriter. - /// - /// # Parameters - /// - /// * `writer_builder` - The IcebergWriterBuilder to use for creating underlying writers - /// * `fanout_enabled` - If true, use FanoutWriter for partitioned tables; otherwise use ClusteredWriter - /// * `schema` - The Iceberg schema reference - /// * `partition_spec` - The partition specification reference - /// - /// # Returns - /// - /// Returns a new TaskWriter instance. - /// - /// # Writer Selection Logic - /// - /// - If partition_spec is unpartitioned: creates UnpartitionedWriter - /// - If partition_spec is partitioned AND fanout_enabled is true: creates FanoutWriter - /// - If partition_spec is partitioned AND fanout_enabled is false: creates ClusteredWriter - /// - /// # Example - /// - /// ```rust,ignore - /// use iceberg::spec::{PartitionSpec, Schema}; - /// use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; - /// use iceberg_datafusion::writer::task_writer::TaskWriter; - /// - /// // Create a TaskWriter for an unpartitioned table - /// let task_writer = TaskWriter::new( - /// data_file_writer_builder, - /// false, // fanout_enabled - /// schema, - /// partition_spec, - /// ); - /// ``` - pub fn try_new( - writer_builder: B, - fanout_enabled: bool, - schema: SchemaRef, - partition_spec: PartitionSpecRef, - ) -> Result { - let writer = if partition_spec.is_unpartitioned() { - SupportedWriter::Unpartitioned(UnpartitionedWriter::new(writer_builder)) - } else if fanout_enabled { - SupportedWriter::Fanout(FanoutWriter::new(writer_builder)) - } else { - SupportedWriter::Clustered(ClusteredWriter::new(writer_builder)) - }; - - // Initialize partition splitter in constructor for partitioned tables - let partition_splitter = if !partition_spec.is_unpartitioned() { - Some( - RecordBatchPartitionSplitter::try_new_with_precomputed_values( - schema.clone(), - partition_spec.clone(), - )?, - ) - } else { - None - }; - - Ok(Self { - writer, - partition_splitter, - }) - } - - /// Write a RecordBatch to the TaskWriter. - /// - /// For partitioned tables, uses the partition splitter to split - /// the batch by partition key and route each partition to the underlying writer. - /// For unpartitioned tables, data is written directly without splitting. - /// - /// # Parameters - /// - /// * `batch` - The RecordBatch to write - /// - /// # Returns - /// - /// Returns `Ok(())` on success, or an error if the write fails. - /// - /// # Errors - /// - /// This method will return an error if: - /// - Splitting the batch by partition fails - /// - Writing to the underlying writer fails - /// - /// # Example - /// - /// ```rust,ignore - /// use arrow_array::RecordBatch; - /// use iceberg_datafusion::writer::task_writer::TaskWriter; - /// - /// // Write a RecordBatch - /// task_writer.write(record_batch).await?; - /// ``` - pub async fn write(&mut self, batch: RecordBatch) -> Result<()> { - match &mut self.writer { - SupportedWriter::Unpartitioned(writer) => { - // Unpartitioned: write directly without splitting - writer.write(batch).await - } - SupportedWriter::Fanout(writer) => { - Self::write_partitioned_batches(writer, &self.partition_splitter, &batch).await - } - SupportedWriter::Clustered(writer) => { - Self::write_partitioned_batches(writer, &self.partition_splitter, &batch).await - } - } - } - - /// Helper method to split and write partitioned data. - /// - /// This method handles the common logic for both FanoutWriter and ClusteredWriter: - /// - Splits the batch by partition key using the partition splitter - /// - Writes each partition to the underlying writer with its corresponding partition key - /// - /// # Parameters - /// - /// * `writer` - The underlying PartitioningWriter (FanoutWriter or ClusteredWriter) - /// * `partition_splitter` - The partition splitter - /// * `batch` - The RecordBatch to write - /// - /// # Returns - /// - /// Returns `Ok(())` on success, or an error if the operation fails. - async fn write_partitioned_batches( - writer: &mut W, - partition_splitter: &Option, - batch: &RecordBatch, - ) -> Result<()> { - // Split batch by partition - let splitter = partition_splitter - .as_ref() - .expect("Partition splitter should be initialized"); - let partitioned_batches = splitter.split(batch)?; - - // Write each partition - for (partition_key, partition_batch) in partitioned_batches { - writer.write(partition_key, partition_batch).await?; - } - - Ok(()) - } - - /// Close the TaskWriter and return all written data files. - /// - /// This method consumes the TaskWriter to prevent further use. - /// - /// # Returns - /// - /// Returns a `Vec` containing all written files, or an error if closing fails. - /// - /// # Errors - /// - /// This method will return an error if: - /// - Closing the underlying writer fails - /// - Any I/O operation fails during the close process - /// - /// # Example - /// - /// ```rust,ignore - /// use iceberg_datafusion::writer::task_writer::TaskWriter; - /// - /// // Close the writer and get all data files - /// let data_files = task_writer.close().await?; - /// ``` - pub async fn close(self) -> Result> { - match self.writer { - SupportedWriter::Unpartitioned(writer) => writer.close().await, - SupportedWriter::Fanout(writer) => writer.close().await, - SupportedWriter::Clustered(writer) => writer.close().await, - } - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; - - use datafusion::arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray, StructArray}; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use iceberg::arrow::PROJECTED_PARTITION_VALUE_COLUMN; - use iceberg::io::FileIO; - use iceberg::spec::{DataFileFormat, NestedField, PartitionSpec, PrimitiveType, Type}; - use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; - use iceberg::writer::file_writer::ParquetWriterBuilder; - use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, - }; - use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - use parquet::file::properties::WriterProperties; - use tempfile::TempDir; - - use super::*; - - fn create_test_schema() -> Result> { - Ok(Arc::new( - iceberg::spec::Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(3, "region", Type::Primitive(PrimitiveType::String)) - .into(), - ]) - .build()?, - )) - } - - fn create_arrow_schema() -> Arc { - Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - ])) - } - - fn create_arrow_schema_with_partition() -> Arc { - let partition_field = Field::new("region", DataType::Utf8, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1000".to_string())]), - ); - let partition_struct_field = Field::new( - PROJECTED_PARTITION_VALUE_COLUMN, - DataType::Struct(vec![partition_field].into()), - false, - ); - - Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - partition_struct_field, - ])) - } - - fn create_writer_builder( - temp_dir: &TempDir, - schema: Arc, - ) -> Result< - DataFileWriterBuilder< - ParquetWriterBuilder, - DefaultLocationGenerator, - DefaultFileNameGenerator, - >, - > { - let file_io = FileIO::new_with_fs(); - let location_gen = DefaultLocationGenerator::with_data_location( - temp_dir.path().to_str().unwrap().to_string(), - ); - let file_name_gen = - DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); - let parquet_writer_builder = - ParquetWriterBuilder::new(WriterProperties::builder().build(), schema); - let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( - parquet_writer_builder, - file_io, - location_gen, - file_name_gen, - ); - Ok(DataFileWriterBuilder::new(rolling_writer_builder)) - } - - #[tokio::test] - async fn test_task_writer_unpartitioned() -> Result<()> { - let temp_dir = TempDir::new()?; - let schema = create_test_schema()?; - let arrow_schema = create_arrow_schema(); - - // Create unpartitioned spec - let partition_spec = Arc::new(PartitionSpec::builder(schema.clone()).build()?); - - let writer_builder = create_writer_builder(&temp_dir, schema.clone())?; - let mut task_writer = TaskWriter::try_new(writer_builder, false, schema, partition_spec)?; - - // Write data - let batch = RecordBatch::try_new(arrow_schema, vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])), - Arc::new(StringArray::from(vec!["US", "EU", "US"])), - ])?; - - task_writer.write(batch).await?; - let data_files = task_writer.close().await?; - - // Verify results - assert!(!data_files.is_empty()); - assert_eq!(data_files[0].record_count(), 3); - - Ok(()) - } - - /// Helper to verify partition data files - fn verify_partition_files( - data_files: &[DataFile], - expected_total: u64, - ) -> HashMap { - let total_records: u64 = data_files.iter().map(|f| f.record_count()).sum(); - assert_eq!(total_records, expected_total, "Total record count mismatch"); - - let mut partition_counts = HashMap::new(); - for data_file in data_files { - let partition_value = data_file.partition(); - let region_literal = partition_value.fields()[0] - .as_ref() - .expect("Partition value should not be null"); - let region = match region_literal - .as_primitive_literal() - .expect("Should be primitive literal") - { - iceberg::spec::PrimitiveLiteral::String(s) => s.clone(), - _ => panic!("Expected string partition value"), - }; - - *partition_counts.entry(region.clone()).or_insert(0) += data_file.record_count(); - - // Verify file path contains partition information - assert!( - data_file.file_path().contains("region="), - "File path should contain partition info" - ); - } - partition_counts - } - - #[tokio::test] - async fn test_task_writer_partitioned_fanout() -> Result<()> { - let temp_dir = TempDir::new()?; - let schema = create_test_schema()?; - let arrow_schema = create_arrow_schema_with_partition(); - - let partition_spec = Arc::new( - PartitionSpec::builder(schema.clone()) - .with_spec_id(1) - .add_partition_field("region", "region", iceberg::spec::Transform::Identity)? - .build()?, - ); - - let writer_builder = create_writer_builder(&temp_dir, schema.clone())?; - let mut task_writer = TaskWriter::try_new(writer_builder, true, schema, partition_spec)?; - - // Create partition column - let partition_field = Field::new("region", DataType::Utf8, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1000".to_string())]), - ); - let partition_values = StringArray::from(vec!["US", "EU", "US", "EU"]); - let partition_struct = StructArray::from(vec![( - Arc::new(partition_field), - Arc::new(partition_values) as ArrayRef, - )]); - - let batch = RecordBatch::try_new(arrow_schema, vec![ - Arc::new(Int32Array::from(vec![1, 2, 3, 4])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie", "Dave"])), - Arc::new(StringArray::from(vec!["US", "EU", "US", "EU"])), - Arc::new(partition_struct), - ])?; - - task_writer.write(batch).await?; - let data_files = task_writer.close().await?; - - let partition_counts = verify_partition_files(&data_files, 4); - assert_eq!(partition_counts.get("US"), Some(&2)); - assert_eq!(partition_counts.get("EU"), Some(&2)); - - Ok(()) - } - - #[tokio::test] - async fn test_task_writer_partitioned_clustered() -> Result<()> { - let temp_dir = TempDir::new()?; - let schema = create_test_schema()?; - let arrow_schema = create_arrow_schema_with_partition(); - - let partition_spec = Arc::new( - PartitionSpec::builder(schema.clone()) - .with_spec_id(1) - .add_partition_field("region", "region", iceberg::spec::Transform::Identity)? - .build()?, - ); - - let writer_builder = create_writer_builder(&temp_dir, schema.clone())?; - let mut task_writer = TaskWriter::try_new(writer_builder, false, schema, partition_spec)?; - - // Create partition column - let partition_field = Field::new("region", DataType::Utf8, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1000".to_string())]), - ); - let partition_values = StringArray::from(vec!["ASIA", "ASIA", "EU", "EU"]); - let partition_struct = StructArray::from(vec![( - Arc::new(partition_field), - Arc::new(partition_values) as ArrayRef, - )]); - - // ClusteredWriter expects data to be pre-sorted by partition - let batch = RecordBatch::try_new(arrow_schema, vec![ - Arc::new(Int32Array::from(vec![1, 2, 3, 4])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie", "Dave"])), - Arc::new(StringArray::from(vec!["ASIA", "ASIA", "EU", "EU"])), - Arc::new(partition_struct), - ])?; - - task_writer.write(batch).await?; - let data_files = task_writer.close().await?; - - let partition_counts = verify_partition_files(&data_files, 4); - assert_eq!(partition_counts.get("ASIA"), Some(&2)); - assert_eq!(partition_counts.get("EU"), Some(&2)); - - Ok(()) - } -} diff --git a/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json b/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json deleted file mode 100644 index a7b47217f5..0000000000 --- a/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "format-version": 2, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "s3://bucket/test/location", - "last-sequence-number": 34, - "last-updated-ms": 1602638573590, - "last-column-id": 3, - "current-schema-id": 1, - "schemas": [ - { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - } - ] - }, - { - "type": "struct", - "schema-id": 1, - "identifier-field-ids": [ - 1, - 2 - ], - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - }, - { - "id": 2, - "name": "y", - "required": true, - "type": "long" - }, - { - "id": 3, - "name": "z", - "required": true, - "type": "long" - } - ] - } - ], - "default-spec-id": 0, - "partition-specs": [ - { - "spec-id": 0, - "fields": [ - { - "name": "x", - "transform": "identity", - "source-id": 1, - "field-id": 1000 - } - ] - } - ], - "last-partition-id": 1000, - "default-sort-order-id": 3, - "sort-orders": [ - { - "order-id": 3, - "fields": [ - { - "transform": "identity", - "source-id": 2, - "direction": "asc", - "null-order": "nulls-first" - }, - { - "transform": "bucket[4]", - "source-id": 3, - "direction": "desc", - "null-order": "nulls-last" - } - ] - } - ], - "properties": {}, - "current-snapshot-id": 3055729675574597004, - "snapshots": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770, - "sequence-number": 0, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/1.avro" - }, - { - "snapshot-id": 3055729675574597004, - "parent-snapshot-id": 3051729675574597004, - "timestamp-ms": 1555100955770, - "sequence-number": 1, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/2.avro", - "schema-id": 1 - } - ], - "snapshot-log": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770 - }, - { - "snapshot-id": 3055729675574597004, - "timestamp-ms": 1555100955770 - } - ], - "metadata-log": [] -} \ No newline at end of file diff --git a/crates/integrations/datafusion/tests/integration_datafusion_test.rs b/crates/integrations/datafusion/tests/integration_datafusion_test.rs deleted file mode 100644 index 9aa6a1a501..0000000000 --- a/crates/integrations/datafusion/tests/integration_datafusion_test.rs +++ /dev/null @@ -1,948 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for Iceberg Datafusion with Hive Metastore. - -use std::collections::HashMap; -use std::sync::Arc; -use std::vec; - -use datafusion::arrow::array::{Array, StringArray, UInt64Array}; -use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; -use datafusion::execution::context::SessionContext; -use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; -use expect_test::expect; -use iceberg::io::LocalFsStorageFactory; -use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; -use iceberg::spec::{ - NestedField, PrimitiveType, Schema, StructType, Transform, Type, UnboundPartitionSpec, -}; -use iceberg::test_utils::check_record_batches; -use iceberg::{ - Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, Result, TableCreation, TableIdent, -}; -use iceberg_datafusion::IcebergCatalogProvider; -use tempfile::TempDir; - -fn temp_path() -> String { - let temp_dir = TempDir::new().unwrap(); - temp_dir.path().to_str().unwrap().to_string() -} - -async fn get_iceberg_catalog() -> MemoryCatalog { - MemoryCatalogBuilder::default() - .with_storage_factory(Arc::new(LocalFsStorageFactory)) - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), temp_path())]), - ) - .await - .unwrap() -} - -fn get_struct_type() -> StructType { - StructType::new(vec![ - NestedField::required(4, "s_foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(5, "s_foo2", Type::Primitive(PrimitiveType::String)).into(), - ]) -} - -async fn set_test_namespace(catalog: &MemoryCatalog, namespace: &NamespaceIdent) -> Result<()> { - let properties = HashMap::new(); - - catalog.create_namespace(namespace, properties).await?; - - Ok(()) -} - -fn get_table_creation( - location: impl ToString, - name: impl ToString, - schema: Option, -) -> Result { - let schema = match schema { - None => Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "foo2", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?, - Some(schema) => schema, - }; - - let creation = TableCreation::builder() - .location(location.to_string()) - .name(name.to_string()) - .properties(HashMap::new()) - .schema(schema) - .build(); - - Ok(creation) -} - -#[tokio::test] -async fn test_provider_plan_stream_schema() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_get_table_schema".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_provider_get_table_schema").unwrap(); - - let table = schema.table("my_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - let expected = [("foo1", &DataType::Int32), ("foo2", &DataType::Utf8)]; - - for (field, exp) in table_schema.fields().iter().zip(expected.iter()) { - assert_eq!(field.name(), exp.0); - assert_eq!(field.data_type(), exp.1); - assert!(!field.is_nullable()) - } - - let df = ctx - .sql("select foo2 from catalog.test_provider_get_table_schema.my_table") - .await - .unwrap(); - - let task_ctx = Arc::new(df.task_ctx()); - let plan = df.create_physical_plan().await.unwrap(); - let stream = plan.execute(1, task_ctx).unwrap(); - - // Ensure both the plan and the stream conform to the same schema - assert_eq!(plan.schema(), stream.schema()); - assert_eq!( - stream.schema().as_ref(), - &ArrowSchema::new(vec![ - Field::new("foo2", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])) - ]), - ); - - Ok(()) -} - -#[tokio::test] -async fn test_provider_list_table_names() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_list_table_names".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_provider_list_table_names").unwrap(); - - let result = schema.table_names(); - - expect![[r#" - [ - "my_table", - "my_table$snapshots", - "my_table$manifests", - "my_table$history", - ] - "#]] - .assert_debug_eq(&result); - - Ok(()) -} - -#[tokio::test] -async fn test_provider_list_schema_names() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_list_schema_names".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - - let expected = ["test_provider_list_schema_names"]; - let result = provider.schema_names(); - - assert!( - expected - .iter() - .all(|item| result.contains(&item.to_string())) - ); - Ok(()) -} - -#[tokio::test] -async fn test_table_projection() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "foo2", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "foo3", Type::Struct(get_struct_type())).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let table_df = ctx.table("catalog.ns.t1").await.unwrap(); - - let records = table_df - .clone() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - // the first column is plan_type, the second column plan string. - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - // the first row is logical_plan, the second row is physical_plan - assert!(s.value(1).contains("projection:[foo1,foo2,foo3]")); - - // datafusion doesn't support query foo3.s_foo1, use foo3 instead - let records = table_df - .select_columns(&["foo1", "foo3"]) - .unwrap() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - assert!( - s.value(1) - .contains("IcebergTableScan projection:[foo1,foo3]") - ); - - Ok(()) -} - -#[tokio::test] -async fn test_table_predict_pushdown() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let records = ctx - .sql("select * from catalog.ns.t1 where (foo > 1 and length(bar) = 1 ) or bar is null") - .await - .unwrap() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - // the first column is plan_type, the second column plan string. - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - // the first row is logical_plan, the second row is physical_plan - let expected = "predicate:[(foo > 1) OR (bar IS NULL)]"; - assert!(s.value(1).trim().contains(expected)); - Ok(()) -} - -#[tokio::test] -async fn test_metadata_table() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let snapshots = ctx - .sql("select * from catalog.ns.t1$snapshots") - .await - .unwrap() - .collect() - .await - .unwrap(); - check_record_batches( - snapshots, - expect![[r#" - Field { "committed_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} }, - Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} }, - Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} }, - Field { "operation": nullable Utf8, metadata: {"PARQUET:field_id": "4"} }, - Field { "manifest_list": nullable Utf8, metadata: {"PARQUET:field_id": "5"} }, - Field { "summary": nullable Map("key_value": non-null Struct("key": non-null Utf8, metadata: {"PARQUET:field_id": "7"}, "value": Utf8, metadata: {"PARQUET:field_id": "8"}), unsorted), metadata: {"PARQUET:field_id": "6"} }"#]], - expect![[r#" - committed_at: PrimitiveArray - [ - ], - snapshot_id: PrimitiveArray - [ - ], - parent_id: PrimitiveArray - [ - ], - operation: StringArray - [ - ], - manifest_list: StringArray - [ - ], - summary: MapArray - [ - ]"#]], - &[], - None, - ); - - let manifests = ctx - .sql("select * from catalog.ns.t1$manifests") - .await - .unwrap() - .collect() - .await - .unwrap(); - check_record_batches( - manifests, - expect![[r#" - Field { "content": Int32, metadata: {"PARQUET:field_id": "14"} }, - Field { "path": Utf8, metadata: {"PARQUET:field_id": "1"} }, - Field { "length": Int64, metadata: {"PARQUET:field_id": "2"} }, - Field { "partition_spec_id": Int32, metadata: {"PARQUET:field_id": "3"} }, - Field { "added_snapshot_id": Int64, metadata: {"PARQUET:field_id": "4"} }, - Field { "added_data_files_count": Int32, metadata: {"PARQUET:field_id": "5"} }, - Field { "existing_data_files_count": Int32, metadata: {"PARQUET:field_id": "6"} }, - Field { "deleted_data_files_count": Int32, metadata: {"PARQUET:field_id": "7"} }, - Field { "added_delete_files_count": Int32, metadata: {"PARQUET:field_id": "15"} }, - Field { "existing_delete_files_count": Int32, metadata: {"PARQUET:field_id": "16"} }, - Field { "deleted_delete_files_count": Int32, metadata: {"PARQUET:field_id": "17"} }, - Field { "partition_summaries": List(non-null Struct("contains_null": non-null Boolean, metadata: {"PARQUET:field_id": "10"}, "contains_nan": Boolean, metadata: {"PARQUET:field_id": "11"}, "lower_bound": Utf8, metadata: {"PARQUET:field_id": "12"}, "upper_bound": Utf8, metadata: {"PARQUET:field_id": "13"}), metadata: {"PARQUET:field_id": "9"}), metadata: {"PARQUET:field_id": "8"} }"#]], - expect![[r#" - content: PrimitiveArray - [ - ], - path: StringArray - [ - ], - length: PrimitiveArray - [ - ], - partition_spec_id: PrimitiveArray - [ - ], - added_snapshot_id: PrimitiveArray - [ - ], - added_data_files_count: PrimitiveArray - [ - ], - existing_data_files_count: PrimitiveArray - [ - ], - deleted_data_files_count: PrimitiveArray - [ - ], - added_delete_files_count: PrimitiveArray - [ - ], - existing_delete_files_count: PrimitiveArray - [ - ], - deleted_delete_files_count: PrimitiveArray - [ - ], - partition_summaries: ListArray - [ - ]"#]], - &[], - None, - ); - - Ok(()) -} - -#[tokio::test] -async fn test_insert_into() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_insert_into".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - // Verify table schema - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_insert_into").unwrap(); - let table = schema.table("my_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - let expected = [("foo1", &DataType::Int32), ("foo2", &DataType::Utf8)]; - for (field, exp) in table_schema.fields().iter().zip(expected.iter()) { - assert_eq!(field.name(), exp.0); - assert_eq!(field.data_type(), exp.1); - assert!(!field.is_nullable()) - } - - // Insert data into the table - let df = ctx - .sql("INSERT INTO catalog.test_insert_into.my_table VALUES (1, 'alan'), (2, 'turing')") - .await - .unwrap(); - - // Verify the insert operation result - let batches = df.collect().await.unwrap(); - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert!( - batch.num_rows() == 1 && batch.num_columns() == 1, - "Results should only have one row and one column that has the number of rows inserted" - ); - // Verify the number of rows inserted - let rows_inserted = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(rows_inserted.value(0), 2); - - // Query the table to verify the inserted data - let df = ctx - .sql("SELECT * FROM catalog.test_insert_into.my_table") - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the data - check_record_batches( - batches, - expect![[r#" - Field { "foo1": Int32, metadata: {"PARQUET:field_id": "1"} }, - Field { "foo2": Utf8, metadata: {"PARQUET:field_id": "2"} }"#]], - expect![[r#" - foo1: PrimitiveArray - [ - 1, - 2, - ], - foo2: StringArray - [ - "alan", - "turing", - ]"#]], - &[], - Some("foo1"), - ); - - Ok(()) -} - -fn get_nested_struct_type() -> StructType { - // Create a nested struct type with: - // - address: STRUCT - // - contact: STRUCT - StructType::new(vec![ - NestedField::optional( - 10, - "address", - Type::Struct(StructType::new(vec![ - NestedField::optional(11, "street", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(12, "city", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(13, "zip", Type::Primitive(PrimitiveType::Int)).into(), - ])), - ) - .into(), - NestedField::optional( - 20, - "contact", - Type::Struct(StructType::new(vec![ - NestedField::optional(21, "email", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(22, "phone", Type::Primitive(PrimitiveType::String)).into(), - ])), - ) - .into(), - ]) -} - -#[tokio::test] -async fn test_insert_into_nested() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_insert_nested".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - let table_name = "nested_table"; - - // Create a schema with nested fields - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "profile", Type::Struct(get_nested_struct_type())).into(), - ]) - .build()?; - - // Create the table with the nested schema - let creation = get_table_creation(temp_path(), table_name, Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - // Verify table schema - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_insert_nested").unwrap(); - let table = schema.table("nested_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - // Verify the schema has the expected structure - assert_eq!(table_schema.fields().len(), 3); - assert_eq!(table_schema.field(0).name(), "id"); - assert_eq!(table_schema.field(1).name(), "name"); - assert_eq!(table_schema.field(2).name(), "profile"); - assert!(matches!( - table_schema.field(2).data_type(), - DataType::Struct(_) - )); - - // In DataFusion, we need to use named_struct to create struct values - // Insert data with nested structs - let insert_sql = r#" - INSERT INTO catalog.test_insert_nested.nested_table - SELECT - 1 as id, - 'Alice' as name, - named_struct( - 'address', named_struct( - 'street', '123 Main St', - 'city', 'San Francisco', - 'zip', 94105 - ), - 'contact', named_struct( - 'email', 'alice@example.com', - 'phone', '555-1234' - ) - ) as profile - UNION ALL - SELECT - 2 as id, - 'Bob' as name, - named_struct( - 'address', named_struct( - 'street', '456 Market St', - 'city', 'San Jose', - 'zip', 95113 - ), - 'contact', named_struct( - 'email', 'bob@example.com', - 'phone', NULL - ) - ) as profile - "#; - - // Execute the insert - let df = ctx.sql(insert_sql).await.unwrap(); - let batches = df.collect().await.unwrap(); - - // Verify the insert operation result - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert!(batch.num_rows() == 1 && batch.num_columns() == 1); - - // Verify the number of rows inserted - let rows_inserted = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(rows_inserted.value(0), 2); - - // Query the table to verify the inserted data - let df = ctx - .sql("SELECT * FROM catalog.test_insert_nested.nested_table ORDER BY id") - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the data - check_record_batches( - batches, - expect![[r#" - Field { "id": Int32, metadata: {"PARQUET:field_id": "1"} }, - Field { "name": Utf8, metadata: {"PARQUET:field_id": "2"} }, - Field { "profile": nullable Struct("address": Struct("street": Utf8, metadata: {"PARQUET:field_id": "6"}, "city": Utf8, metadata: {"PARQUET:field_id": "7"}, "zip": Int32, metadata: {"PARQUET:field_id": "8"}), metadata: {"PARQUET:field_id": "4"}, "contact": Struct("email": Utf8, metadata: {"PARQUET:field_id": "9"}, "phone": Utf8, metadata: {"PARQUET:field_id": "10"}), metadata: {"PARQUET:field_id": "5"}), metadata: {"PARQUET:field_id": "3"} }"#]], - expect![[r#" - id: PrimitiveArray - [ - 1, - 2, - ], - name: StringArray - [ - "Alice", - "Bob", - ], - profile: StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "address" (Struct([Field { name: "street", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "6"} }, Field { name: "city", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "7"} }, Field { name: "zip", data_type: Int32, nullable: true, metadata: {"PARQUET:field_id": "8"} }])) - StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "street" (Utf8) - StringArray - [ - "123 Main St", - "456 Market St", - ] - -- child 1: "city" (Utf8) - StringArray - [ - "San Francisco", - "San Jose", - ] - -- child 2: "zip" (Int32) - PrimitiveArray - [ - 94105, - 95113, - ] - ] - -- child 1: "contact" (Struct([Field { name: "email", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "9"} }, Field { name: "phone", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "10"} }])) - StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "email" (Utf8) - StringArray - [ - "alice@example.com", - "bob@example.com", - ] - -- child 1: "phone" (Utf8) - StringArray - [ - "555-1234", - null, - ] - ] - ]"#]], - &[], - Some("id"), - ); - - // Query with explicit field access to verify nested data - let df = ctx - .sql( - r#" - SELECT - id, - name, - profile.address.street, - profile.address.city, - profile.address.zip, - profile.contact.email, - profile.contact.phone - FROM catalog.test_insert_nested.nested_table - ORDER BY id - "#, - ) - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the flattened data - check_record_batches( - batches, - expect![[r#" - Field { "id": Int32, metadata: {"PARQUET:field_id": "1"} }, - Field { "name": Utf8, metadata: {"PARQUET:field_id": "2"} }, - Field { "catalog.test_insert_nested.nested_table.profile[address][street]": nullable Utf8, metadata: {"PARQUET:field_id": "6"} }, - Field { "catalog.test_insert_nested.nested_table.profile[address][city]": nullable Utf8, metadata: {"PARQUET:field_id": "7"} }, - Field { "catalog.test_insert_nested.nested_table.profile[address][zip]": nullable Int32, metadata: {"PARQUET:field_id": "8"} }, - Field { "catalog.test_insert_nested.nested_table.profile[contact][email]": nullable Utf8, metadata: {"PARQUET:field_id": "9"} }, - Field { "catalog.test_insert_nested.nested_table.profile[contact][phone]": nullable Utf8, metadata: {"PARQUET:field_id": "10"} }"#]], - expect![[r#" - id: PrimitiveArray - [ - 1, - 2, - ], - name: StringArray - [ - "Alice", - "Bob", - ], - catalog.test_insert_nested.nested_table.profile[address][street]: StringArray - [ - "123 Main St", - "456 Market St", - ], - catalog.test_insert_nested.nested_table.profile[address][city]: StringArray - [ - "San Francisco", - "San Jose", - ], - catalog.test_insert_nested.nested_table.profile[address][zip]: PrimitiveArray - [ - 94105, - 95113, - ], - catalog.test_insert_nested.nested_table.profile[contact][email]: StringArray - [ - "alice@example.com", - "bob@example.com", - ], - catalog.test_insert_nested.nested_table.profile[contact][phone]: StringArray - [ - "555-1234", - null, - ]"#]], - &[], - Some("id"), - ); - - Ok(()) -} - -#[tokio::test] -async fn test_insert_into_partitioned() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_partitioned_write".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - // Create a schema with a partition column - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(3, "value", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - // Create partition spec with identity transform on category - let partition_spec = UnboundPartitionSpec::builder() - .with_spec_id(0) - .add_partition_field(2, "category", Transform::Identity)? - .build(); - - // Create the partitioned table - let creation = TableCreation::builder() - .name("partitioned_table".to_string()) - .location(temp_path()) - .schema(schema) - .partition_spec(partition_spec) - .properties(HashMap::new()) - .build(); - - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - // Insert data with multiple partition values in a single batch - let df = ctx - .sql( - r#" - INSERT INTO catalog.test_partitioned_write.partitioned_table - VALUES - (1, 'electronics', 'laptop'), - (2, 'electronics', 'phone'), - (3, 'books', 'novel'), - (4, 'books', 'textbook'), - (5, 'clothing', 'shirt') - "#, - ) - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - let rows_inserted = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(rows_inserted.value(0), 5); - - // Query the table to verify data - let df = ctx - .sql("SELECT * FROM catalog.test_partitioned_write.partitioned_table ORDER BY id") - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Verify the data - note that _partition column should NOT be present - check_record_batches( - batches, - expect![[r#" - Field { "id": Int32, metadata: {"PARQUET:field_id": "1"} }, - Field { "category": Utf8, metadata: {"PARQUET:field_id": "2"} }, - Field { "value": Utf8, metadata: {"PARQUET:field_id": "3"} }"#]], - expect![[r#" - id: PrimitiveArray - [ - 1, - 2, - 3, - 4, - 5, - ], - category: StringArray - [ - "electronics", - "electronics", - "books", - "books", - "clothing", - ], - value: StringArray - [ - "laptop", - "phone", - "novel", - "textbook", - "shirt", - ]"#]], - &[], - Some("id"), - ); - - // Verify that data files exist under correct partition paths - let table_ident = TableIdent::new(namespace.clone(), "partitioned_table".to_string()); - let table = client.load_table(&table_ident).await?; - let table_location = table.metadata().location(); - let file_io = table.file_io(); - - // List files under each expected partition path - let electronics_path = format!("{table_location}/data/category=electronics"); - let books_path = format!("{table_location}/data/category=books"); - let clothing_path = format!("{table_location}/data/category=clothing"); - - // Verify partition directories exist and contain data files - assert!( - file_io.exists(&electronics_path).await?, - "Expected partition directory: {electronics_path}" - ); - assert!( - file_io.exists(&books_path).await?, - "Expected partition directory: {books_path}" - ); - assert!( - file_io.exists(&clothing_path).await?, - "Expected partition directory: {clothing_path}" - ); - - Ok(()) -} diff --git a/crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json b/crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json deleted file mode 100644 index 0dc89de58c..0000000000 --- a/crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "format-version": 2, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "s3://bucket/test/location", - "last-sequence-number": 34, - "last-updated-ms": 1602638573590, - "last-column-id": 3, - "current-schema-id": 1, - "schemas": [ - { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - } - ] - }, - { - "type": "struct", - "schema-id": 1, - "identifier-field-ids": [ - 1, - 2 - ], - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - }, - { - "id": 2, - "name": "y", - "required": true, - "type": "long", - "doc": "comment" - }, - { - "id": 3, - "name": "z", - "required": true, - "type": "long" - } - ] - } - ], - "default-spec-id": 0, - "partition-specs": [ - { - "spec-id": 0, - "fields": [ - { - "name": "x", - "transform": "identity", - "source-id": 1, - "field-id": 1000 - } - ] - } - ], - "last-partition-id": 1000, - "default-sort-order-id": 3, - "sort-orders": [ - { - "order-id": 3, - "fields": [ - { - "transform": "identity", - "source-id": 2, - "direction": "asc", - "null-order": "nulls-first" - }, - { - "transform": "bucket[4]", - "source-id": 3, - "direction": "desc", - "null-order": "nulls-last" - } - ] - } - ], - "properties": {}, - "current-snapshot-id": 3055729675574597004, - "snapshots": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770, - "sequence-number": 0, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/1.avro" - }, - { - "snapshot-id": 3055729675574597004, - "parent-snapshot-id": 3051729675574597004, - "timestamp-ms": 1555100955770, - "sequence-number": 1, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/2.avro", - "schema-id": 1 - } - ], - "snapshot-log": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770 - }, - { - "snapshot-id": 3055729675574597004, - "timestamp-ms": 1555100955770 - } - ], - "metadata-log": [] -} \ No newline at end of file diff --git a/crates/integrations/playground/Cargo.toml b/crates/integrations/playground/Cargo.toml deleted file mode 100644 index 60137e2ff8..0000000000 --- a/crates/integrations/playground/Cargo.toml +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -description = "Apache iceberg client" -edition.workspace = true -homepage.workspace = true -license.workspace = true -name = "iceberg-playground" -publish = false -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[dependencies] -anyhow = { workspace = true } -clap = { workspace = true } -datafusion = { workspace = true } -datafusion-cli = { workspace = true } -dirs = { workspace = true } -fs-err = { workspace = true } -home = { workspace = true } -iceberg = { workspace = true } -iceberg-catalog-rest = { workspace = true } -iceberg-datafusion = { workspace = true } -mimalloc = { workspace = true } -stacker = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } - -[package.metadata.cargo-machete] -# These dependencies are added to ensure minimal dependency version -ignored = ["stacker", "mimalloc", "home"] - -[lints] -workspace = true diff --git a/crates/integrations/playground/DEPENDENCIES.rust.tsv b/crates/integrations/playground/DEPENDENCIES.rust.tsv deleted file mode 100644 index 49af43d19f..0000000000 --- a/crates/integrations/playground/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,517 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anstream@0.6.21 X X -anstream@1.0.0 X X -anstyle@1.0.14 X X -anstyle-parse@0.2.7 X X -anstyle-parse@1.0.0 X X -anstyle-query@1.1.5 X X -anstyle-wincon@3.0.11 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -ar_archive_writer@0.5.1 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@58.1.0 X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-csv@58.1.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-json@58.1.0 X -arrow-ord@58.3.0 X -arrow-row@58.1.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-compression@0.4.41 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.15 X -aws-credential-types@1.2.14 X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -aws-runtime@1.7.2 X -aws-sdk-sso@1.97.0 X -aws-sdk-ssooidc@1.99.0 X -aws-sdk-sts@1.101.0 X -aws-sigv4@1.4.2 X -aws-smithy-async@1.2.14 X -aws-smithy-http@0.63.6 X -aws-smithy-http-client@1.1.12 X -aws-smithy-json@0.62.5 X -aws-smithy-observability@0.2.6 X -aws-smithy-query@0.60.15 X -aws-smithy-runtime@1.10.3 X -aws-smithy-runtime-api@1.11.6 X -aws-smithy-types@1.4.7 X -aws-smithy-xml@0.60.15 X -aws-types@1.3.14 X -backon@1.6.0 X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -blake2@0.10.6 X X -blake3@1.8.3 X X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bytes-utils@0.1.4 X X -bzip2@0.6.1 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -cfg_aliases@0.2.1 X -chacha20@0.10.0 X X -chrono@0.4.44 X X -chrono-tz@0.10.4 X X -cipher@0.4.4 X X -clap@4.6.0 X X -clap_builder@4.6.0 X X -clap_derive@4.6.0 X X -clap_lex@1.1.0 X X -clipboard-win@5.4.1 X -cmake@0.1.57 X X -colorchoice@1.0.5 X X -comfy-table@7.2.2 X -compression-codecs@0.4.37 X X -compression-core@0.4.31 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.4.2 X X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -cpufeatures@0.3.0 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -csv@1.4.0 X X -csv-core@0.1.13 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -datafusion@53.1.0 X -datafusion-catalog@53.1.0 X -datafusion-catalog-listing@53.1.0 X -datafusion-cli@53.1.0 X -datafusion-common@53.1.0 X -datafusion-common-runtime@53.1.0 X -datafusion-datasource@53.1.0 X -datafusion-datasource-arrow@53.1.0 X -datafusion-datasource-avro@53.1.0 X -datafusion-datasource-csv@53.1.0 X -datafusion-datasource-json@53.1.0 X -datafusion-datasource-parquet@53.1.0 X -datafusion-doc@53.1.0 X -datafusion-execution@53.1.0 X -datafusion-expr@53.1.0 X -datafusion-expr-common@53.1.0 X -datafusion-functions@53.1.0 X -datafusion-functions-aggregate@53.1.0 X -datafusion-functions-aggregate-common@53.1.0 X -datafusion-functions-nested@53.1.0 X -datafusion-functions-table@53.1.0 X -datafusion-functions-window@53.1.0 X -datafusion-functions-window-common@53.1.0 X -datafusion-macros@53.1.0 X -datafusion-optimizer@53.1.0 X -datafusion-physical-expr@53.1.0 X -datafusion-physical-expr-adapter@53.1.0 X -datafusion-physical-expr-common@53.1.0 X -datafusion-physical-optimizer@53.1.0 X -datafusion-physical-plan@53.1.0 X -datafusion-pruning@53.1.0 X -datafusion-session@53.1.0 X -datafusion-sql@53.1.0 X -deranged@0.5.8 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -dirs@6.0.0 X X -dirs-sys@0.5.0 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dunce@1.0.5 X X X -either@1.15.0 X X -endian-type@0.1.2 X -env_filter@1.0.0 X X -env_logger@0.11.9 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -error-code@3.3.2 X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -fd-lock@4.0.4 X X -find-msvc-tools@0.1.9 X X -fixedbitset@0.5.7 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -foldhash@0.2.0 X -form_urlencoded@1.2.2 X X -fs-err@3.3.0 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -h2@0.4.13 X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@0.2.12 X X -http@1.4.0 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -humantime@2.3.0 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-rest@0.10.0 X -iceberg-datafusion@0.10.0 X -iceberg-playground@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -is_terminal_polyfill@1.70.2 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libbz2-rs-sys@0.2.2 X -libc@0.2.183 X X -liblzma@0.4.6 X X -liblzma-sys@0.4.5 X X -libm@0.2.16 X -libmimalloc-sys@0.1.44 X -libredox@0.1.14 X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.10.6 X X -memchr@2.8.0 X X -mimalloc@0.1.48 X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nibble_vec@0.1.0 X -nix@0.30.1 X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.2.0 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -object@0.37.3 X X -object_store@0.13.2 X X -once_cell@1.21.4 X X -once_cell_polyfill@1.70.2 X X -opaque-debug@0.3.1 X X -openssl-probe@0.2.1 X X -option-ext@0.2.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -petgraph@0.8.3 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -psm@0.1.30 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -radix_trie@0.2.1 X -rand@0.10.1 X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.10.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.18 X -redox_users@0.5.2 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -rustyline@17.0.2 X -ryu@1.0.23 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_spanned@0.6.9 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.8 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -siphasher@1.0.2 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -sqlparser@0.61.0 X -sqlparser_derive@0.5.0 X -stable_deref_trait@1.2.1 X X -stacker@0.1.23 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.27.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -time@0.3.47 X X -time-core@0.1.8 X X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-rustls@0.26.4 X X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -toml@0.8.23 X X -toml_datetime@0.6.11 X X -toml_edit@0.22.27 X X -toml_write@0.1.2 X X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.2 X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -utf8parse@0.2.2 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -vsimd@0.8.0 X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.4.2 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -winnow@0.7.15 X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xmlparser@0.13.6 X X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/playground/README.md b/crates/integrations/playground/README.md deleted file mode 100644 index c471724598..0000000000 --- a/crates/integrations/playground/README.md +++ /dev/null @@ -1,25 +0,0 @@ - - - -# Introduction - -Iceberg Playground (`iceberg-playground`) is a small command line utility that runs SQL queries against tables, -which is backed by the DataFusion engine. - diff --git a/crates/integrations/playground/src/catalog.rs b/crates/integrations/playground/src/catalog.rs deleted file mode 100644 index e815cf720b..0000000000 --- a/crates/integrations/playground/src/catalog.rs +++ /dev/null @@ -1,204 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::path::Path; -use std::sync::Arc; - -use anyhow::anyhow; -use datafusion::catalog::{CatalogProvider, CatalogProviderList}; -use fs_err::read_to_string; -use iceberg::CatalogBuilder; -use iceberg::memory::MemoryCatalogBuilder; -use iceberg_catalog_rest::RestCatalogBuilder; -use iceberg_datafusion::IcebergCatalogProvider; -use toml::{Table as TomlTable, Value}; - -const CONFIG_NAME_CATALOGS: &str = "catalogs"; - -#[derive(Debug)] -pub struct IcebergCatalogList { - catalogs: HashMap>, -} - -impl IcebergCatalogList { - pub async fn parse(path: &Path) -> anyhow::Result { - let toml_table: TomlTable = toml::from_str(&read_to_string(path)?)?; - Self::parse_table(&toml_table).await - } - - pub async fn parse_table(configs: &TomlTable) -> anyhow::Result { - if let Value::Array(catalogs_config) = - configs.get(CONFIG_NAME_CATALOGS).ok_or_else(|| { - anyhow::Error::msg(format!("{CONFIG_NAME_CATALOGS} entry not found in config")) - })? - { - let mut catalogs = HashMap::with_capacity(catalogs_config.len()); - for config in catalogs_config { - if let Value::Table(table_config) = config { - let (name, catalog_provider) = - IcebergCatalogList::parse_one(table_config).await?; - catalogs.insert(name, catalog_provider); - } else { - return Err(anyhow!("{CONFIG_NAME_CATALOGS} entry must be a table")); - } - } - Ok(Self { catalogs }) - } else { - Err(anyhow!("{CONFIG_NAME_CATALOGS} must be an array of table!")) - } - } - - async fn parse_one( - config: &TomlTable, - ) -> anyhow::Result<(String, Arc)> { - let name = config - .get("name") - .ok_or_else(|| anyhow::anyhow!("name not found for catalog"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("name is not string"))?; - - let r#type = config - .get("type") - .ok_or_else(|| anyhow::anyhow!("type not found for catalog"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("type is not string"))?; - - let catalog_config = config - .get("config") - .ok_or_else(|| anyhow::anyhow!("config not found for catalog {name}"))? - .as_table() - .ok_or_else(|| anyhow::anyhow!("config is not table for catalog {name}"))?; - - // parse all config into props - let mut props = HashMap::with_capacity(catalog_config.len()); - for (key, value) in catalog_config { - let value_str = value - .as_str() - .ok_or_else(|| anyhow::anyhow!("props {key} is not string"))?; - props.insert(key.to_string(), value_str.to_string()); - } - - // Create catalog based on type using the appropriate builder - let catalog: Arc = match r#type { - "rest" => Arc::new(RestCatalogBuilder::default().load(name, props).await?), - "memory" => Arc::new(MemoryCatalogBuilder::default().load(name, props).await?), - _ => { - return Err(anyhow::anyhow!( - "Unsupported catalog type: '{type}'. Supported types: rest, memory" - )); - } - }; - - Ok(( - name.to_string(), - Arc::new(IcebergCatalogProvider::try_new(catalog).await?), - )) - } -} - -impl CatalogProviderList for IcebergCatalogList { - fn register_catalog( - &self, - _name: String, - _catalog: Arc, - ) -> Option> { - tracing::error!("Registering catalog is not supported yet"); - None - } - - fn catalog_names(&self) -> Vec { - self.catalogs.keys().cloned().collect() - } - - fn catalog(&self, name: &str) -> Option> { - self.catalogs - .get(name) - .map(|c| c.clone() as Arc) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_parse_memory_catalog() { - let config = r#" - [[catalogs]] - name = "test_memory" - type = "memory" - [catalogs.config] - warehouse = "/tmp/test-warehouse" - "#; - - let toml_table: TomlTable = toml::from_str(config).unwrap(); - let catalog_list = IcebergCatalogList::parse_table(&toml_table).await.unwrap(); - - assert!( - catalog_list - .catalog_names() - .contains(&"test_memory".to_string()) - ); - assert!(catalog_list.catalog("test_memory").is_some()); - } - - #[tokio::test] - async fn test_parse_unsupported_catalog_type() { - let config = r#" - [[catalogs]] - name = "test_hive" - type = "hive" - [catalogs.config] - uri = "thrift://localhost:9083" - "#; - - let toml_table: TomlTable = toml::from_str(config).unwrap(); - let result = IcebergCatalogList::parse_table(&toml_table).await; - - assert!(result.is_err()); - let err_msg = result.unwrap_err().to_string(); - assert!(err_msg.contains("Unsupported catalog type")); - assert!(err_msg.contains("hive")); - assert!(err_msg.contains("rest, memory")); - } - - #[tokio::test] - async fn test_catalog_names() { - let config = r#" - [[catalogs]] - name = "catalog_one" - type = "memory" - [catalogs.config] - warehouse = "/tmp/warehouse1" - - [[catalogs]] - name = "catalog_two" - type = "memory" - [catalogs.config] - warehouse = "/tmp/warehouse2" - "#; - - let toml_table: TomlTable = toml::from_str(config).unwrap(); - let catalog_list = IcebergCatalogList::parse_table(&toml_table).await.unwrap(); - - let names = catalog_list.catalog_names(); - assert_eq!(names.len(), 2); - assert!(names.contains(&"catalog_one".to_string())); - assert!(names.contains(&"catalog_two".to_string())); - } -} diff --git a/crates/integrations/playground/src/lib.rs b/crates/integrations/playground/src/lib.rs deleted file mode 100644 index de62f4b143..0000000000 --- a/crates/integrations/playground/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![doc = include_str!("../README.md")] -pub const ICEBERG_PLAYGROUND_VERSION: &str = env!("CARGO_PKG_VERSION"); - -mod catalog; -pub use catalog::*; diff --git a/crates/integrations/playground/src/main.rs b/crates/integrations/playground/src/main.rs deleted file mode 100644 index 94068bb558..0000000000 --- a/crates/integrations/playground/src/main.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::path::PathBuf; -use std::process::ExitCode; -use std::str::FromStr; -use std::sync::Arc; - -use clap::Parser; -use datafusion::execution::runtime_env::RuntimeEnvBuilder; -use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion_cli::exec; -use datafusion_cli::object_storage::instrumented::InstrumentedObjectStoreRegistry; -use datafusion_cli::print_format::PrintFormat; -use datafusion_cli::print_options::{MaxRows, PrintOptions}; -use iceberg_playground::{ICEBERG_PLAYGROUND_VERSION, IcebergCatalogList}; - -#[derive(Debug, Parser, PartialEq)] -#[clap(author, version, about, long_about= None)] -struct Args { - #[clap( - short = 'r', - long, - help = "Parse catalog config instead of using ~/.icebergrc" - )] - rc: Option, - - #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)] - format: PrintFormat, - - #[clap( - short, - long, - help = "Reduce printing other than the results and work quietly" - )] - quiet: bool, - - #[clap( - long, - help = "The max number of rows to display for 'Table' format\n[possible values: numbers(0/10/...), inf(no limit)]", - default_value = "40" - )] - maxrows: MaxRows, - - #[clap(long, help = "Enables console syntax highlighting")] - color: bool, -} - -#[tokio::main] -/// Calls [`main_inner`], then handles printing errors and returning the correct exit code -pub async fn main() -> ExitCode { - tracing_subscriber::fmt::init(); - - if let Err(e) = main_inner().await { - println!("Error: {e}"); - return ExitCode::FAILURE; - } - - ExitCode::SUCCESS -} - -async fn main_inner() -> anyhow::Result<()> { - let args = Args::parse(); - - if !args.quiet { - println!("ICEBERG PLAYGROUND v{ICEBERG_PLAYGROUND_VERSION}"); - } - - let session_config = SessionConfig::from_env()?.with_information_schema(true); - - let rt_builder = RuntimeEnvBuilder::new(); - - let runtime_env = rt_builder.build_arc()?; - - // enable dynamic file query - let ctx = SessionContext::new_with_config_rt(session_config, runtime_env).enable_url_table(); - ctx.refresh_catalogs().await?; - - let mut print_options = PrintOptions { - format: args.format, - quiet: args.quiet, - maxrows: args.maxrows, - color: args.color, - instrumented_registry: Arc::new(InstrumentedObjectStoreRegistry::new()), - }; - - let rc = match args.rc { - Some(file) => PathBuf::from_str(&file)?, - None => dirs::home_dir() - .map(|h| h.join(".icebergrc")) - .ok_or_else(|| anyhow::anyhow!("cannot find home directory"))?, - }; - - let catalogs = Arc::new(IcebergCatalogList::parse(&rc).await?); - ctx.register_catalog_list(catalogs); - - Ok(exec::exec_from_repl(&ctx, &mut print_options).await?) -} diff --git a/crates/sqllogictest/Cargo.toml b/crates/sqllogictest/Cargo.toml deleted file mode 100644 index 37f6a663f2..0000000000 --- a/crates/sqllogictest/Cargo.toml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -name = "iceberg-sqllogictest" -publish = false -repository = { workspace = true } -rust-version = { workspace = true } -version = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -async-trait = { workspace = true } -datafusion = { workspace = true } -datafusion-sqllogictest = { workspace = true } -enum-ordinalize = { workspace = true } -env_logger = { workspace = true } -iceberg = { workspace = true } -iceberg-datafusion = { workspace = true } -indicatif = { workspace = true } -log = { workspace = true } -serde = { workspace = true } -sqllogictest = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } - -[dev-dependencies] -libtest-mimic = { workspace = true } - -[[test]] -harness = false -name = "sqllogictests" -path = "tests/sqllogictests.rs" - -[package.metadata.cargo-machete] -# These dependencies are added to ensure minimal dependency version -ignored = ["enum-ordinalize"] - -[lints] -workspace = true diff --git a/crates/sqllogictest/DEPENDENCIES.rust.tsv b/crates/sqllogictest/DEPENDENCIES.rust.tsv deleted file mode 100644 index d50ca9d2df..0000000000 --- a/crates/sqllogictest/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,481 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anstream@0.6.21 X X -anstream@1.0.0 X X -anstyle@1.0.14 X X -anstyle-parse@0.2.7 X X -anstyle-parse@1.0.0 X X -anstyle-query@1.1.5 X X -anstyle-wincon@3.0.11 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -ar_archive_writer@0.5.1 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@58.1.0 X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-csv@58.1.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-json@58.1.0 X -arrow-ord@58.3.0 X -arrow-row@58.1.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-compression@0.4.41 X X -async-lock@3.4.2 X X -async-recursion@1.1.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -blake2@0.10.6 X X -blake3@1.8.3 X X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bzip2@0.6.1 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -chrono-tz@0.10.4 X X -cipher@0.4.4 X X -clap@4.6.0 X X -clap_builder@4.6.0 X X -clap_derive@4.6.0 X X -clap_lex@1.1.0 X X -colorchoice@1.0.5 X X -comfy-table@7.2.2 X -compression-codecs@0.4.37 X X -compression-core@0.4.31 X X -concurrent-queue@2.5.0 X X -console@0.16.3 X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.4.2 X X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -csv@1.4.0 X X -csv-core@0.1.13 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -datafusion@53.1.0 X -datafusion-catalog@53.1.0 X -datafusion-catalog-listing@53.1.0 X -datafusion-common@53.1.0 X -datafusion-common-runtime@53.1.0 X -datafusion-datasource@53.1.0 X -datafusion-datasource-arrow@53.1.0 X -datafusion-datasource-avro@53.1.0 X -datafusion-datasource-csv@53.1.0 X -datafusion-datasource-json@53.1.0 X -datafusion-datasource-parquet@53.1.0 X -datafusion-doc@53.1.0 X -datafusion-execution@53.1.0 X -datafusion-expr@53.1.0 X -datafusion-expr-common@53.1.0 X -datafusion-functions@53.1.0 X -datafusion-functions-aggregate@53.1.0 X -datafusion-functions-aggregate-common@53.1.0 X -datafusion-functions-nested@53.1.0 X -datafusion-functions-table@53.1.0 X -datafusion-functions-window@53.1.0 X -datafusion-functions-window-common@53.1.0 X -datafusion-macros@53.1.0 X -datafusion-optimizer@53.1.0 X -datafusion-physical-expr@53.1.0 X -datafusion-physical-expr-adapter@53.1.0 X -datafusion-physical-expr-common@53.1.0 X -datafusion-physical-optimizer@53.1.0 X -datafusion-physical-plan@53.1.0 X -datafusion-pruning@53.1.0 X -datafusion-session@53.1.0 X -datafusion-spark@53.1.0 X -datafusion-sql@53.1.0 X -datafusion-sqllogictest@53.1.0 X -datafusion-substrait@53.1.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dyn-clone@1.0.20 X X -educe@0.6.0 X -either@1.15.0 X X -encode_unicode@1.0.0 X X -enum-ordinalize@4.4.1 X -enum-ordinalize-derive@4.4.1 X -env_filter@1.0.0 X X -env_logger@0.11.9 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -escape8259@0.5.3 X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -fixedbitset@0.5.7 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -foldhash@0.2.0 X -form_urlencoded@1.2.2 X X -fs-err@3.3.0 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -humantime@2.3.0 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-datafusion@0.10.0 X -iceberg-sqllogictest@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -indicatif@0.18.4 X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -is_terminal_polyfill@1.70.2 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libbz2-rs-sys@0.2.2 X -libc@0.2.183 X X -liblzma@0.4.6 X X -liblzma-sys@0.4.5 X X -libm@0.2.16 X -libtest-mimic@0.8.2 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.10.6 X X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -multimap@0.10.1 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -object@0.37.3 X X -object_store@0.13.2 X X -once_cell@1.21.4 X X -once_cell_polyfill@1.70.2 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -owo-colors@4.3.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -pbjson@0.8.0 X -pbjson-build@0.8.0 X -pbjson-types@0.8.0 X -percent-encoding@2.3.2 X X -petgraph@0.8.3 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -prost@0.14.3 X -prost-build@0.14.3 X -prost-derive@0.14.3 X -prost-types@0.14.3 X -psm@0.1.30 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.8.5 X X -rand@0.9.4 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -regress@0.10.5 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -schemars@0.8.22 X -schemars_derive@0.8.22 X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_derive_internals@0.29.1 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_spanned@0.6.9 X X -serde_tokenstream@0.2.3 X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -serde_yaml@0.9.34+deprecated X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -similar@2.7.0 X -siphasher@1.0.2 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -sqllogictest@0.29.1 X X -sqlparser@0.61.0 X -sqlparser_derive@0.5.0 X -stable_deref_trait@1.2.1 X X -stacker@0.1.23 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subst@0.3.8 X X -substrait@0.62.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.27.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -toml@0.8.23 X X -toml_datetime@0.6.11 X X -toml_edit@0.22.27 X X -toml_write@0.1.2 X X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -typify@0.5.0 X -typify-impl@0.5.0 X -typify-macro@0.5.0 X -unicode-ident@1.0.24 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.1.14 X X -unicode-width@0.2.2 X X -unit-prefix@0.5.2 X -universal-hash@0.5.1 X X -unsafe-libyaml@0.2.11 X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -utf8parse@0.2.2 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -winnow@0.7.15 X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/sqllogictest/README.md b/crates/sqllogictest/README.md deleted file mode 100644 index ddcfe851c5..0000000000 --- a/crates/sqllogictest/README.md +++ /dev/null @@ -1,35 +0,0 @@ - - -This crate contains a suite of [sqllogictest](https://crates.io/crates/sqllogictest) tests that are used to validate [iceberg-rust](https://github.com/apache/iceberg-rust). - -## Running the tests - -Just run the following command: - -```bash -cargo test -``` - -## Sql Engines - -The tests are run against the following sql engines: - -* [Apache datafusion](https://crates.io/crates/datafusion) -* [Apache spark](https://github.com/apache/spark) \ No newline at end of file diff --git a/crates/sqllogictest/src/engine/datafusion.rs b/crates/sqllogictest/src/engine/datafusion.rs deleted file mode 100644 index 0b52d50e53..0000000000 --- a/crates/sqllogictest/src/engine/datafusion.rs +++ /dev/null @@ -1,204 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use datafusion::catalog::CatalogProvider; -use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion_sqllogictest::DataFusion; -use iceberg::encryption::kms::MemoryKmsClientFactory; -use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; -use iceberg::spec::{ - NestedField, PrimitiveType, Schema, TableProperties, Transform, Type, UnboundPartitionSpec, -}; -use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation}; -use iceberg_datafusion::IcebergCatalogProvider; -use indicatif::ProgressBar; - -use crate::engine::{DatafusionCatalogConfig, EngineRunner, run_slt_with_runner}; -use crate::error::Result; - -const TEST_MASTER_KEY_ID: &str = "test-master-key"; - -pub struct DataFusionEngine { - test_data_path: PathBuf, - session_context: SessionContext, -} - -#[async_trait::async_trait] -impl EngineRunner for DataFusionEngine { - async fn run_slt_file(&mut self, path: &Path) -> Result<()> { - let ctx = self.session_context.clone(); - let testdata = self.test_data_path.clone(); - - let runner = sqllogictest::Runner::new({ - move || { - let ctx = ctx.clone(); - let testdata = testdata.clone(); - async move { - // Everything here is owned; no `self` capture. - Ok(DataFusion::new(ctx, testdata, ProgressBar::new(100))) - } - } - }); - - run_slt_with_runner(runner, path).await - } -} - -impl DataFusionEngine { - pub async fn new(catalog_config: Option) -> Result { - let session_config = SessionConfig::new() - .with_target_partitions(4) - .with_information_schema(true); - let ctx = SessionContext::new_with_config(session_config); - ctx.register_catalog( - "default", - Self::create_catalog(catalog_config.as_ref()).await?, - ); - - Ok(Self { - test_data_path: PathBuf::from("testdata"), - session_context: ctx, - }) - } - - async fn create_catalog( - _catalog_config: Option<&DatafusionCatalogConfig>, - ) -> anyhow::Result> { - // TODO: Use catalog_config to load different catalog types via iceberg-catalog-loader - // See: https://github.com/apache/iceberg-rust/issues/1780 - let kms_factory = MemoryKmsClientFactory::new(); - kms_factory.add_master_key(TEST_MASTER_KEY_ID)?; - - let catalog = MemoryCatalogBuilder::default() - .with_kms_client_factory(Arc::new(kms_factory)) - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://".to_string(), - )]), - ) - .await?; - - // Create a test namespace for INSERT INTO tests - let namespace = NamespaceIdent::new("default".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - // Create partitioned test table (unpartitioned tables are now created via SQL) - Self::create_partitioned_table(&catalog, &namespace).await?; - Self::create_binary_table(&catalog, &namespace).await?; - Self::create_encrypted_table(&catalog, &namespace).await?; - - Ok(Arc::new( - IcebergCatalogProvider::try_new(Arc::new(catalog)).await?, - )) - } - - /// Create a partitioned test table with id, category, and value columns - /// Partitioned by category using identity transform - /// TODO: this can be removed when we support CREATE EXTERNAL TABLE - async fn create_partitioned_table( - catalog: &impl Catalog, - namespace: &NamespaceIdent, - ) -> anyhow::Result<()> { - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "value", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - let partition_spec = UnboundPartitionSpec::builder() - .with_spec_id(0) - .add_partition_field(2, "category", Transform::Identity)? - .build(); - - catalog - .create_table( - namespace, - TableCreation::builder() - .name("test_partitioned_table".to_string()) - .schema(schema) - .partition_spec(partition_spec) - .build(), - ) - .await?; - - Ok(()) - } - - /// Create a test table with binary type column - /// Used for testing binary predicate pushdown - /// TODO: this can be removed when we support CREATE TABLE - async fn create_binary_table( - catalog: &impl Catalog, - namespace: &NamespaceIdent, - ) -> anyhow::Result<()> { - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Binary)).into(), - ]) - .build()?; - - catalog - .create_table( - namespace, - TableCreation::builder() - .name("test_binary_table".to_string()) - .schema(schema) - .build(), - ) - .await?; - - Ok(()) - } - - async fn create_encrypted_table( - catalog: &impl Catalog, - namespace: &NamespaceIdent, - ) -> anyhow::Result<()> { - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - catalog - .create_table( - namespace, - TableCreation::builder() - .name("test_encrypted_round_trip".to_string()) - .schema(schema) - .format_version(iceberg::spec::FormatVersion::V3) - .properties([( - TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), - TEST_MASTER_KEY_ID.to_string(), - )]) - .build(), - ) - .await?; - - Ok(()) - } -} diff --git a/crates/sqllogictest/src/engine/mod.rs b/crates/sqllogictest/src/engine/mod.rs deleted file mode 100644 index a276671401..0000000000 --- a/crates/sqllogictest/src/engine/mod.rs +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod datafusion; - -use std::collections::HashMap; -use std::path::Path; - -use anyhow::anyhow; -use serde::Deserialize; -use sqllogictest::{AsyncDB, MakeConnection, Runner, parse_file}; - -use crate::engine::datafusion::DataFusionEngine; -use crate::error::{Error, Result}; - -/// Configuration for the catalog used by the DataFusion engine -#[derive(Debug, Clone, Deserialize)] -pub struct DatafusionCatalogConfig { - /// Catalog type: "memory", "rest", "glue", "hms", "s3tables", "sql" - #[serde(rename = "type")] - pub catalog_type: String, - /// Catalog properties passed to the catalog loader - #[serde(default)] - pub props: HashMap, -} - -/// Engine configuration as a tagged enum -#[derive(Debug, Clone, Deserialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub enum EngineConfig { - Datafusion { - #[serde(default)] - catalog: Option, - }, -} - -#[async_trait::async_trait] -pub trait EngineRunner: Send { - async fn run_slt_file(&mut self, path: &Path) -> Result<()>; -} - -pub async fn load_engine_runner(config: EngineConfig) -> Result> { - match config { - EngineConfig::Datafusion { catalog } => Ok(Box::new(DataFusionEngine::new(catalog).await?)), - } -} - -pub async fn run_slt_with_runner( - mut runner: Runner, - step_slt_file: impl AsRef, -) -> Result<()> -where - D: AsyncDB + Send + 'static, - M: MakeConnection + Send + 'static, -{ - let path = step_slt_file.as_ref().canonicalize()?; - let records = parse_file(&path).map_err(|e| Error(anyhow!("parsing slt file failed: {e}")))?; - - for record in records { - if let Err(err) = runner.run_async(record).await { - return Err(Error(anyhow!("SLT record execution failed: {err}"))); - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use crate::engine::{DatafusionCatalogConfig, EngineConfig, load_engine_runner}; - - #[test] - fn test_deserialize_engine_config() { - let input = r#"type = "datafusion""#; - - let config: EngineConfig = toml::from_str(input).unwrap(); - assert!(matches!(config, EngineConfig::Datafusion { catalog: None })); - } - - #[test] - fn test_deserialize_engine_config_with_catalog() { - let input = r#" - type = "datafusion" - - [catalog] - type = "rest" - - [catalog.props] - uri = "http://localhost:8181" - "#; - - let config: EngineConfig = toml::from_str(input).unwrap(); - match config { - EngineConfig::Datafusion { catalog: Some(cat) } => { - assert_eq!(cat.catalog_type, "rest"); - assert_eq!( - cat.props.get("uri"), - Some(&"http://localhost:8181".to_string()) - ); - } - _ => panic!("Expected Datafusion with catalog"), - } - } - - #[test] - fn test_deserialize_catalog_config() { - let input = r#" - type = "memory" - - [props] - warehouse = "file:///tmp/warehouse" - "#; - - let config: DatafusionCatalogConfig = toml::from_str(input).unwrap(); - assert_eq!(config.catalog_type, "memory"); - assert_eq!( - config.props.get("warehouse"), - Some(&"file:///tmp/warehouse".to_string()) - ); - } - - #[tokio::test] - async fn test_load_datafusion() { - let config = EngineConfig::Datafusion { catalog: None }; - - let result = load_engine_runner(config).await; - assert!(result.is_ok()); - } -} diff --git a/crates/sqllogictest/src/error.rs b/crates/sqllogictest/src/error.rs deleted file mode 100644 index 2bf5a09d6b..0000000000 --- a/crates/sqllogictest/src/error.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fmt::{Debug, Display, Formatter}; - -pub struct Error(pub anyhow::Error); - -pub type Result = std::result::Result; - -impl Debug for Error { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } -} - -impl Display for Error { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - self.0.source() - } -} - -impl From for Error { - fn from(value: anyhow::Error) -> Self { - Self(value) - } -} - -impl From for Error { - fn from(value: std::io::Error) -> Self { - Self(value.into()) - } -} diff --git a/crates/sqllogictest/src/lib.rs b/crates/sqllogictest/src/lib.rs deleted file mode 100644 index 34a3c654aa..0000000000 --- a/crates/sqllogictest/src/lib.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// This lib contains codes copied from -// [Apache Datafusion](https://github.com/apache/datafusion/tree/main/datafusion/sqllogictest) - -mod engine; -pub mod error; -pub mod schedule; diff --git a/crates/sqllogictest/src/schedule.rs b/crates/sqllogictest/src/schedule.rs deleted file mode 100644 index 29e37c5c72..0000000000 --- a/crates/sqllogictest/src/schedule.rs +++ /dev/null @@ -1,255 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::fs::read_to_string; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, anyhow}; -use serde::{Deserialize, Serialize}; -use tracing::info; - -use crate::engine::{EngineConfig, EngineRunner, load_engine_runner}; - -/// Raw configuration parsed from the schedule TOML file -#[derive(Debug, Clone, Deserialize)] -pub struct ScheduleConfig { - /// Engine name to engine configuration - pub engines: HashMap, - /// List of test steps to run - pub steps: Vec, -} - -pub struct Schedule { - /// Engine names to engine instances - engines: HashMap>, - /// List of test steps to run - steps: Vec, - /// Path of the schedule file - schedule_file: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Step { - /// Engine name - engine: String, - /// Stl file path - slt: String, -} - -impl Schedule { - pub fn new( - engines: HashMap>, - steps: Vec, - schedule_file: String, - ) -> Self { - Self { - engines, - steps, - schedule_file, - } - } - - pub async fn from_file>(path: P) -> anyhow::Result { - let path_str = path.as_ref().to_string_lossy().to_string(); - let content = read_to_string(path)?; - - let config: ScheduleConfig = toml::from_str(&content) - .with_context(|| format!("Failed to parse schedule file: {path_str}"))?; - - let engines = Self::instantiate_engines(config.engines).await?; - - Ok(Self::new(engines, config.steps, path_str)) - } - - /// Instantiate engine runners from their configurations - async fn instantiate_engines( - configs: HashMap, - ) -> anyhow::Result>> { - let mut engines = HashMap::new(); - - for (name, config) in configs { - let engine = load_engine_runner(config).await?; - engines.insert(name, engine); - } - - Ok(engines) - } - - pub async fn run(mut self) -> anyhow::Result<()> { - info!("Starting test run with schedule: {}", self.schedule_file); - - for (idx, step) in self.steps.iter().enumerate() { - info!( - "Running step {}/{}, using engine {}, slt file path: {}", - idx + 1, - self.steps.len(), - &step.engine, - &step.slt - ); - - let engine = self - .engines - .get_mut(&step.engine) - .ok_or_else(|| anyhow!("Engine {} not found", step.engine))?; - - let step_sql_path = PathBuf::from(format!( - "{}/testdata/slts/{}", - env!("CARGO_MANIFEST_DIR"), - step.slt - )); - - engine.run_slt_file(&step_sql_path).await?; - - info!( - "Completed step {}/{}, engine {}, slt file path: {}", - idx + 1, - self.steps.len(), - &step.engine, - &step.slt - ); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::engine::EngineConfig; - use crate::schedule::ScheduleConfig; - - #[test] - fn test_deserialize_schedule_config() { - let input = r#" - [engines] - df = { type = "datafusion" } - - [[steps]] - engine = "df" - slt = "test.slt" - "#; - - let config: ScheduleConfig = toml::from_str(input).unwrap(); - - assert_eq!(config.engines.len(), 1); - assert!(config.engines.contains_key("df")); - assert!(matches!(config.engines["df"], EngineConfig::Datafusion { - catalog: None - })); - assert_eq!(config.steps.len(), 1); - assert_eq!(config.steps[0].engine, "df"); - assert_eq!(config.steps[0].slt, "test.slt"); - } - - #[test] - fn test_deserialize_multiple_steps() { - let input = r#" - [engines] - datafusion = { type = "datafusion" } - - [[steps]] - engine = "datafusion" - slt = "test.slt" - - [[steps]] - engine = "datafusion" - slt = "test2.slt" - "#; - - let config: ScheduleConfig = toml::from_str(input).unwrap(); - - assert_eq!(config.steps.len(), 2); - assert_eq!(config.steps[0].engine, "datafusion"); - assert_eq!(config.steps[0].slt, "test.slt"); - assert_eq!(config.steps[1].engine, "datafusion"); - assert_eq!(config.steps[1].slt, "test2.slt"); - } - - #[test] - fn test_deserialize_with_catalog_config() { - let input = r#" - [engines.df] - type = "datafusion" - - [engines.df.catalog] - type = "rest" - - [engines.df.catalog.props] - uri = "http://localhost:8181" - - [[steps]] - engine = "df" - slt = "test.slt" - "#; - - let config: ScheduleConfig = toml::from_str(input).unwrap(); - - match &config.engines["df"] { - EngineConfig::Datafusion { catalog: Some(cat) } => { - assert_eq!(cat.catalog_type, "rest"); - assert_eq!( - cat.props.get("uri"), - Some(&"http://localhost:8181".to_string()) - ); - } - _ => panic!("Expected Datafusion with catalog config"), - } - } - - #[test] - fn test_deserialize_missing_engine_type() { - let input = r#" - [engines] - df = { } - - [[steps]] - engine = "df" - slt = "test.slt" - "#; - - let result: Result = toml::from_str(input); - assert!(result.is_err()); - } - - #[test] - fn test_deserialize_invalid_engine_type() { - let input = r#" - [engines] - df = { type = "unknown_engine" } - - [[steps]] - engine = "df" - slt = "test.slt" - "#; - - let result: Result = toml::from_str(input); - assert!(result.is_err()); - } - - #[test] - fn test_deserialize_missing_step_fields() { - let input = r#" - [engines] - df = { type = "datafusion" } - - [[steps]] - "#; - - let result: Result = toml::from_str(input); - assert!(result.is_err()); - } -} diff --git a/crates/sqllogictest/testdata/schedules/df_basic_queries.toml b/crates/sqllogictest/testdata/schedules/df_basic_queries.toml deleted file mode 100644 index c0d5439d30..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_basic_queries.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/basic_queries.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml b/crates/sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml deleted file mode 100644 index de0093acb2..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/binary_predicate_pushdown.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml b/crates/sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml deleted file mode 100644 index 5b4f63dc7c..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/boolean_predicate_pushdown.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_create_table.toml b/crates/sqllogictest/testdata/schedules/df_create_table.toml deleted file mode 100644 index 1935c9fbd7..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_create_table.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/create_table.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_drop_table.toml b/crates/sqllogictest/testdata/schedules/df_drop_table.toml deleted file mode 100644 index 8a6804dff7..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_drop_table.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/drop_table.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml b/crates/sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml deleted file mode 100644 index 231649ef3b..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/encrypted_insert_round_trip.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_insert_and_like.toml b/crates/sqllogictest/testdata/schedules/df_insert_and_like.toml deleted file mode 100644 index 0c89a0426a..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_insert_and_like.toml +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/insert_into.slt" - -[[steps]] -engine = "df" -slt = "df_test/like_predicate_pushdown.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_scan_all_types.toml b/crates/sqllogictest/testdata/schedules/df_scan_all_types.toml deleted file mode 100644 index cb97fec641..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_scan_all_types.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/scan_all_types.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_show_tables.toml b/crates/sqllogictest/testdata/schedules/df_show_tables.toml deleted file mode 100644 index 5fdd8eec93..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_show_tables.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/show_tables.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml b/crates/sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml deleted file mode 100644 index 849bba50a2..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/timestamp_predicate_pushdown.slt" diff --git a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt b/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt deleted file mode 100644 index a5ca4de46a..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt +++ /dev/null @@ -1,224 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test basic SELECT queries with ORDER BY, LIMIT, and various clauses -# This covers the functionality from the original datafusion.rs integration test - -# Create a test table with multiple columns for query testing -statement ok -CREATE TABLE default.default.query_test_table ( - id INT NOT NULL, - name STRING, - score DOUBLE, - category STRING -) - -# Insert test data -query I -INSERT INTO default.default.query_test_table VALUES - (1, 'Alice', 95.5, 'A'), - (2, 'Bob', 87.0, 'B'), - (3, 'Charlie', 92.5, 'A'), - (4, 'Diana', 78.5, 'C'), - (5, 'Eve', 88.0, 'B'), - (6, 'Frank', 91.0, 'A'), - (7, 'Grace', 85.5, 'B'), - (8, 'Henry', 79.0, 'C'), - (9, 'Ivy', 96.5, 'A'), - (10, 'Jack', 82.0, 'C') ----- -10 - -# Verify EXPLAIN shows limit is pushed down to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.query_test_table LIMIT 3 ----- -logical_plan -01)Limit: skip=0, fetch=3 -02)--TableScan: default.default.query_test_table projection=[id, name, score, category], fetch=3 -physical_plan -01)GlobalLimitExec: skip=0, fetch=3 -02)--CooperativeExec -03)----IcebergTableScan projection:[id,name,score,category] predicate:[] limit:[3] - -# Test SELECT * with ORDER BY and LIMIT -query ITRT -SELECT * FROM default.default.query_test_table ORDER BY id LIMIT 3 ----- -1 Alice 95.5 A -2 Bob 87 B -3 Charlie 92.5 A - -# Test ORDER BY DESC with LIMIT -query ITRT -SELECT * FROM default.default.query_test_table ORDER BY score DESC LIMIT 5 ----- -9 Ivy 96.5 A -1 Alice 95.5 A -3 Charlie 92.5 A -6 Frank 91 A -5 Eve 88 B - -# Test ORDER BY with multiple columns -query ITR -SELECT id, name, score FROM default.default.query_test_table ORDER BY category, score DESC ----- -9 Ivy 96.5 -1 Alice 95.5 -3 Charlie 92.5 -6 Frank 91 -5 Eve 88 -2 Bob 87 -7 Grace 85.5 -10 Jack 82 -8 Henry 79 -4 Diana 78.5 - -# Test LIMIT with OFFSET -query IT -SELECT id, name FROM default.default.query_test_table ORDER BY id LIMIT 3 OFFSET 5 ----- -6 Frank -7 Grace -8 Henry - -# Test WHERE clause with ORDER BY -query ITR -SELECT id, name, score FROM default.default.query_test_table WHERE score > 90 ORDER BY score DESC ----- -9 Ivy 96.5 -1 Alice 95.5 -3 Charlie 92.5 -6 Frank 91 - -# Test WHERE clause with string comparison -query ITT rowsort -SELECT id, name, category FROM default.default.query_test_table WHERE category = 'A' ----- -1 Alice A -3 Charlie A -6 Frank A -9 Ivy A - -# Test aggregation with GROUP BY -query TIR rowsort -SELECT category, COUNT(*), AVG(score) FROM default.default.query_test_table GROUP BY category ----- -A 4 93.875 -B 3 86.833333333333 -C 3 79.833333333333 - -# Test aggregation with HAVING clause -query TI rowsort -SELECT category, COUNT(*) as cnt FROM default.default.query_test_table GROUP BY category HAVING COUNT(*) >= 4 ----- -A 4 - -# Test DISTINCT -query T rowsort -SELECT DISTINCT category FROM default.default.query_test_table ----- -A -B -C - -# Test COUNT with DISTINCT -query I -SELECT COUNT(DISTINCT category) FROM default.default.query_test_table ----- -3 - -# Test MIN and MAX -query RR -SELECT MIN(score), MAX(score) FROM default.default.query_test_table ----- -78.5 96.5 - -# Test SUM -query R -SELECT SUM(score) FROM default.default.query_test_table ----- -875.5 - -# Test combined WHERE, ORDER BY, and LIMIT -query ITR -SELECT id, name, score FROM default.default.query_test_table -WHERE category IN ('A', 'B') AND score > 85 -ORDER BY score DESC -LIMIT 4 ----- -9 Ivy 96.5 -1 Alice 95.5 -3 Charlie 92.5 -6 Frank 91 - -# Test aliasing -query IT -SELECT id AS user_id, name AS user_name FROM default.default.query_test_table ORDER BY user_id LIMIT 2 ----- -1 Alice -2 Bob - -# Test expression in SELECT -query ITR -SELECT id, name, score * 1.1 AS adjusted_score FROM default.default.query_test_table ORDER BY id LIMIT 3 ----- -1 Alice 105.05 -2 Bob 95.7 -3 Charlie 101.75 - -# Test BETWEEN -query IT rowsort -SELECT id, name FROM default.default.query_test_table WHERE score BETWEEN 85 AND 92 ----- -2 Bob -5 Eve -6 Frank -7 Grace - -# Test NOT IN -query IT rowsort -SELECT id, name FROM default.default.query_test_table WHERE category NOT IN ('A') ----- -10 Jack -2 Bob -4 Diana -5 Eve -7 Grace -8 Henry - -# Test LIKE pattern matching -query IT rowsort -SELECT id, name FROM default.default.query_test_table WHERE name LIKE 'A%' ----- -1 Alice - -# Test OR condition -query ITT rowsort -SELECT id, name, category FROM default.default.query_test_table WHERE category = 'A' OR score > 90 ----- -1 Alice A -3 Charlie A -6 Frank A -9 Ivy A - -# Test AND condition -query ITR rowsort -SELECT id, name, score FROM default.default.query_test_table WHERE category = 'A' AND score > 93 ----- -1 Alice 95.5 -9 Ivy 96.5 diff --git a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt deleted file mode 100644 index aa68ab2762..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test that binary predicates are pushed down to IcebergTableScan -# This validates that ScalarValue::Binary is correctly converted to Datum::binary - -# Verify EXPLAIN shows binary predicate is pushed down to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.test_binary_table WHERE data = X'0102' ----- -logical_plan -01)Filter: default.default.test_binary_table.data = LargeBinary("1,2") -02)--TableScan: default.default.test_binary_table projection=[id, data], partial_filters=[default.default.test_binary_table.data = LargeBinary("1,2")] -physical_plan -01)FilterExec: data@1 = 0102 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,data] predicate:[data = 0102] - -# Verify empty result from empty table -query I? -SELECT * FROM default.default.test_binary_table WHERE data = X'0102' ----- - -# Insert test data -query I -INSERT INTO default.default.test_binary_table VALUES (1, X'0102') ----- -1 - -# Verify binary predicate filtering works -query I? -SELECT * FROM default.default.test_binary_table WHERE data = X'0102' ----- -1 0102 - -# Insert more rows with different binary values -query I -INSERT INTO default.default.test_binary_table VALUES (2, X'0304'), (3, X'0102'), (4, X'0506') ----- -3 - -# Verify binary predicate filters correctly -query I? rowsort -SELECT * FROM default.default.test_binary_table WHERE data = X'0102' ----- -1 0102 -3 0102 - -# Verify different binary value -query I? -SELECT * FROM default.default.test_binary_table WHERE data = X'0506' ----- -4 0506 diff --git a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt deleted file mode 100644 index 496f719261..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt +++ /dev/null @@ -1,124 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Create test table with boolean column -statement ok -CREATE TABLE default.default.test_boolean_table (id INT NOT NULL, is_active BOOLEAN, description STRING) - -# Insert test data into test_boolean_table -statement ok -INSERT INTO default.default.test_boolean_table VALUES - (1, true, 'Active user'), - (2, false, 'Inactive user'), - (3, true, 'Premium member'), - (4, false, 'Trial expired'), - (5, true, 'Verified account'), - (6, NULL, 'Pending verification') - -# Verify boolean equality predicate is pushed down to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.test_boolean_table WHERE is_active = true ----- -logical_plan -01)Filter: default.default.test_boolean_table.is_active -02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[default.default.test_boolean_table.is_active] -physical_plan -01)FilterExec: is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = true] - -# Query with is_active = true -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active = true ----- -1 true Active user -3 true Premium member -5 true Verified account - -# Verify boolean false predicate is pushed down to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.test_boolean_table WHERE is_active = false ----- -logical_plan -01)Filter: NOT default.default.test_boolean_table.is_active -02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] -physical_plan -01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] - -# Query with is_active = false -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active = false ----- -2 false Inactive user -4 false Trial expired - -# Verify boolean NOT EQUAL predicate is pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_boolean_table WHERE is_active != true ----- -logical_plan -01)Filter: NOT default.default.test_boolean_table.is_active -02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] -physical_plan -01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] - -# Query with is_active != true (includes false and NULL) -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active != true ----- -2 false Inactive user -4 false Trial expired - -# Test combined boolean predicates with AND -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active = true AND id > 2 ----- -3 true Premium member -5 true Verified account - -# Test combined boolean predicates with OR -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active = true OR id = 2 ----- -1 true Active user -2 false Inactive user -3 true Premium member -5 true Verified account - -# Test IS NULL on boolean column -query ITT -SELECT * FROM default.default.test_boolean_table WHERE is_active IS NULL ----- -6 NULL Pending verification - -# Test IS NOT NULL on boolean column -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active IS NOT NULL ----- -1 true Active user -2 false Inactive user -3 true Premium member -4 false Trial expired -5 true Verified account - -# Clean up: Drop the test table -statement ok -DROP TABLE default.default.test_boolean_table diff --git a/crates/sqllogictest/testdata/slts/df_test/create_table.slt b/crates/sqllogictest/testdata/slts/df_test/create_table.slt deleted file mode 100644 index 2eab1b6bab..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/create_table.slt +++ /dev/null @@ -1,90 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test CREATE TABLE with explicit schema -statement ok -CREATE TABLE default.default.empty_table (id INT NOT NULL, name STRING) - -# Verify the empty table exists and has correct schema -query IT rowsort -SELECT * FROM default.default.empty_table ----- - -# Insert data into the created table -query I -INSERT INTO default.default.empty_table VALUES (1, 'Alice') ----- -1 - -# Verify the inserted data -query IT rowsort -SELECT * FROM default.default.empty_table ----- -1 Alice - -# Insert multiple rows -query I -INSERT INTO default.default.empty_table VALUES (2, 'Bob'), (3, 'Charlie') ----- -2 - -# Verify all rows -query IT rowsort -SELECT * FROM default.default.empty_table ----- -1 Alice -2 Bob -3 Charlie - -# Test CREATE TABLE with different column types -statement ok -CREATE TABLE default.default.typed_table (id BIGINT NOT NULL, value DOUBLE, flag BOOLEAN) - -# Verify the typed table exists -query IDT rowsort -SELECT * FROM default.default.typed_table ----- - -# Insert data with different types -query I -INSERT INTO default.default.typed_table VALUES (100, 3.14, true), (200, 2.71, false) ----- -2 - -# Verify typed data -query IDT rowsort -SELECT * FROM default.default.typed_table ----- -100 3.14 true -200 2.71 false - -# Test CREATE TABLE with nullable columns -statement ok -CREATE TABLE default.default.nullable_table (id INT NOT NULL, optional_name STRING) - -# Insert with NULL value -query I -INSERT INTO default.default.nullable_table VALUES (1, 'Value'), (2, NULL) ----- -2 - -# Verify NULL handling -query IT rowsort -SELECT * FROM default.default.nullable_table ----- -1 Value -2 NULL diff --git a/crates/sqllogictest/testdata/slts/df_test/drop_table.slt b/crates/sqllogictest/testdata/slts/df_test/drop_table.slt deleted file mode 100644 index c721c84e11..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/drop_table.slt +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test DROP TABLE functionality - -# Create a table to drop -statement ok -CREATE TABLE default.default.table_to_drop (id INT NOT NULL, name STRING) - -# Verify the table exists -query IT rowsort -SELECT * FROM default.default.table_to_drop ----- - -# Insert some data -query I -INSERT INTO default.default.table_to_drop VALUES (1, 'Test') ----- -1 - -# Verify data exists -query IT rowsort -SELECT * FROM default.default.table_to_drop ----- -1 Test - -# Drop the table -statement ok -DROP TABLE default.default.table_to_drop - -# Verify the table no longer exists (should error) -statement error -SELECT * FROM default.default.table_to_drop - -# Test DROP TABLE IF EXISTS on non-existent table (should not error) -statement ok -DROP TABLE IF EXISTS default.default.nonexistent_table - -# Create another table for IF EXISTS test -statement ok -CREATE TABLE default.default.another_table (id INT NOT NULL) - -# Drop with IF EXISTS -statement ok -DROP TABLE IF EXISTS default.default.another_table - -# Verify it's gone -statement error -SELECT * FROM default.default.another_table diff --git a/crates/sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt b/crates/sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt deleted file mode 100644 index f1760242f0..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -query II rowsort -SELECT * FROM default.default.test_encrypted_round_trip ----- - -query I -INSERT INTO default.default.test_encrypted_round_trip VALUES (1, 'Alice'), (2, 'Bob') ----- -2 - -query II rowsort -SELECT * FROM default.default.test_encrypted_round_trip ----- -1 Alice -2 Bob diff --git a/crates/sqllogictest/testdata/slts/df_test/insert_into.slt b/crates/sqllogictest/testdata/slts/df_test/insert_into.slt deleted file mode 100644 index 56fc1fefb1..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/insert_into.slt +++ /dev/null @@ -1,153 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Create unpartitioned test table -statement ok -CREATE TABLE default.default.test_unpartitioned_table (id INT NOT NULL, name STRING) - -# Verify the table is initially empty -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table ----- - -# Insert zero rows and verify it returns count 0 without creating a snapshot -query I -INSERT INTO default.default.test_unpartitioned_table SELECT * FROM (VALUES (1, 'Nobody')) AS t(id, name) WHERE false ----- -0 - -query I -SELECT COUNT(*) FROM default.default.test_unpartitioned_table ----- -0 - -query ?????? -SELECT * FROM default.default.test_unpartitioned_table$snapshots ----- - -# Insert a single row and verify the count -query I -INSERT INTO default.default.test_unpartitioned_table VALUES (1, 'Alice') ----- -1 - -# Verify the inserted row -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table ----- -1 Alice - -# Insert multiple rows and verify the count -query I -INSERT INTO default.default.test_unpartitioned_table VALUES (2, 'Bob'), (3, 'Charlie') ----- -2 - -# Verify all rows -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table ----- -1 Alice -2 Bob -3 Charlie - -# Insert with NULL value and verify the count -query I -INSERT INTO default.default.test_unpartitioned_table VALUES (4, NULL) ----- -1 - -# Verify NULL handling -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table ----- -1 Alice -2 Bob -3 Charlie -4 NULL - -# Test partitioned table - verify initially empty -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table ----- - -# Insert zero rows into a partitioned table and verify it returns count 0 without creating a snapshot -query I -INSERT INTO default.default.test_partitioned_table SELECT * FROM (VALUES (1, 'electronics', 'laptop')) AS t(id, category, value) WHERE false ----- -0 - -query I -SELECT COUNT(*) FROM default.default.test_partitioned_table ----- -0 - -query ?????? -SELECT * FROM default.default.test_partitioned_table$snapshots ----- - -# Insert single row into partitioned table -query I -INSERT INTO default.default.test_partitioned_table VALUES (1, 'electronics', 'laptop') ----- -1 - -# Verify the inserted row in partitioned table -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table ----- -1 electronics laptop - -# Insert multiple rows with different partition values -query I -INSERT INTO default.default.test_partitioned_table VALUES (2, 'electronics', 'phone'), (3, 'books', 'novel'), (4, 'books', 'textbook'), (5, 'clothing', 'shirt') ----- -4 - -# Verify all rows in partitioned table -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table ----- -1 electronics laptop -2 electronics phone -3 books novel -4 books textbook -5 clothing shirt - -# Insert with NULL value in optional column -query I -INSERT INTO default.default.test_partitioned_table VALUES (6, 'electronics', NULL) ----- -1 - -# Verify NULL handling in partitioned table -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table ----- -1 electronics laptop -2 electronics phone -3 books novel -4 books textbook -5 clothing shirt -6 electronics NULL - -# Verify partition filtering works -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table WHERE category = 'books' ----- -3 books novel -4 books textbook diff --git a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt deleted file mode 100644 index 3d8b151aa9..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt +++ /dev/null @@ -1,109 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test LIKE predicate pushdown for StartsWith patterns -# This validates that LIKE 'prefix%' patterns are converted to StartsWith operator -# Note: Iceberg's StartsWith operator is case-sensitive - -# Note: test_unpartitioned_table already contains data from insert_into.slt: -# (1, 'Alice'), (2, 'Bob'), (3, 'Charlie'), (4, NULL) -# We'll insert additional test data for LIKE testing -query I -INSERT INTO default.default.test_unpartitioned_table VALUES (5, 'alice'), (6, 'Albert'), (7, 'Bobby'), (8, 'Carol') ----- -4 - -# Test basic LIKE prefix pattern pushdown - verify predicate is pushed to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'Al%' ----- -logical_plan -01)Filter: default.default.test_unpartitioned_table.name LIKE Utf8("Al%") -02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name LIKE Utf8("Al%")] -physical_plan -01)FilterExec: name@1 LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,name] predicate:[name STARTS WITH "Al"] - -# Test LIKE filtering with case-sensitive match -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'Al%' ----- -1 Alice -6 Albert - -# Test NOT LIKE prefix pattern pushdown -query TT -EXPLAIN SELECT * FROM default.default.test_unpartitioned_table WHERE name NOT LIKE 'Al%' ----- -logical_plan -01)Filter: default.default.test_unpartitioned_table.name NOT LIKE Utf8("Al%") -02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name NOT LIKE Utf8("Al%")] -physical_plan -01)FilterExec: name@1 NOT LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,name] predicate:[name NOT STARTS WITH "Al"] - -# Test NOT LIKE filtering -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name NOT LIKE 'Al%' ----- -2 Bob -3 Charlie -5 alice -7 Bobby -8 Carol - -# Test case sensitivity - lowercase 'al%' should not match 'Alice' or 'Albert' -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'al%' ----- -5 alice - -# Test LIKE with empty prefix (matches everything except NULL) -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE '%' ----- -1 Alice -2 Bob -3 Charlie -5 alice -6 Albert -7 Bobby -8 Carol - -# Test LIKE with single character prefix -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'B%' ----- -2 Bob -7 Bobby - -# Test LIKE combined with other predicates -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'Al%' AND id > 1 ----- -6 Albert - -# Test LIKE in OR expression -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'Al%' OR name LIKE 'Bo%' ----- -1 Alice -2 Bob -6 Albert -7 Bobby diff --git a/crates/sqllogictest/testdata/slts/df_test/scan_all_types.slt b/crates/sqllogictest/testdata/slts/df_test/scan_all_types.slt deleted file mode 100644 index d93e461333..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/scan_all_types.slt +++ /dev/null @@ -1,128 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test scanning all primitive types supported by Iceberg -# This tests INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, BOOLEAN, DATE, TIMESTAMP - -# Create a table with all primitive types -statement ok -CREATE TABLE default.default.all_primitive_types ( - col_int INT NOT NULL, - col_long BIGINT NOT NULL, - col_float FLOAT NOT NULL, - col_double DOUBLE NOT NULL, - col_decimal DECIMAL(10, 2), - col_string STRING, - col_boolean BOOLEAN -) - -# Verify the table is initially empty -query IIRRTTT rowsort -SELECT * FROM default.default.all_primitive_types ----- - -# Insert test data with various primitive types -query I -INSERT INTO default.default.all_primitive_types VALUES - (1, 100, 1.5, 2.5, 123.45, 'hello', true), - (2, 200, 2.5, 3.5, 234.56, 'world', false), - (3, 300, 3.5, 4.5, 345.67, NULL, true), - (-1, -100, -1.5, -2.5, -123.45, 'negative', false), - (0, 0, 0.0, 0.0, 0.00, 'zero', NULL) ----- -5 - -# Verify all inserted rows -query IIRRTTT rowsort -SELECT * FROM default.default.all_primitive_types ----- --1 -100 -1.5 -2.5 -123.45 negative false -0 0 0 0 0 zero NULL -1 100 1.5 2.5 123.45 hello true -2 200 2.5 3.5 234.56 world false -3 300 3.5 4.5 345.67 NULL true - -# Test selecting specific columns with string filter -query IT rowsort -SELECT col_int, col_string FROM default.default.all_primitive_types WHERE col_string = 'hello' ----- -1 hello - -# Test numeric comparisons -query IR rowsort -SELECT col_int, col_decimal FROM default.default.all_primitive_types WHERE col_decimal > 200.00 ----- -2 234.56 -3 345.67 - -# Test filtering on int column -query IIRRTTT rowsort -SELECT * FROM default.default.all_primitive_types WHERE col_int >= 0 ORDER BY col_int ----- -0 0 0 0 0 zero NULL -1 100 1.5 2.5 123.45 hello true -2 200 2.5 3.5 234.56 world false -3 300 3.5 4.5 345.67 NULL true - -# Test filtering with NULL -query IT rowsort -SELECT col_int, col_string FROM default.default.all_primitive_types WHERE col_string IS NOT NULL ----- --1 negative -0 zero -1 hello -2 world - -# Test aggregations on numeric types -query IIRR -SELECT COUNT(*), SUM(col_int), AVG(col_float), MAX(col_double) FROM default.default.all_primitive_types ----- -5 5 1.2 4.5 - -# Create a table with date and timestamp types -statement ok -CREATE TABLE default.default.temporal_types ( - id INT NOT NULL, - col_date DATE, - col_timestamp TIMESTAMP -) - -# Insert temporal data -query I -INSERT INTO default.default.temporal_types VALUES - (1, '2024-01-15', '2024-01-15 10:30:00'), - (2, '2024-06-20', '2024-06-20 14:45:30'), - (3, '2023-12-31', '2023-12-31 23:59:59'), - (4, NULL, NULL) ----- -4 - -# Verify temporal data -query IDP rowsort -SELECT * FROM default.default.temporal_types ----- -1 2024-01-15 2024-01-15T10:30:00 -2 2024-06-20 2024-06-20T14:45:30 -3 2023-12-31 2023-12-31T23:59:59 -4 NULL NULL - -# Test date comparisons -query ID rowsort -SELECT id, col_date FROM default.default.temporal_types WHERE col_date >= '2024-01-01' ----- -1 2024-01-15 -2 2024-06-20 diff --git a/crates/sqllogictest/testdata/slts/df_test/show_tables.slt b/crates/sqllogictest/testdata/slts/df_test/show_tables.slt deleted file mode 100644 index 29cc3cd7e6..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/show_tables.slt +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -query TTTT rowsort -SHOW TABLES ----- -datafusion information_schema columns VIEW -datafusion information_schema df_settings VIEW -datafusion information_schema parameters VIEW -datafusion information_schema routines VIEW -datafusion information_schema schemata VIEW -datafusion information_schema tables VIEW -datafusion information_schema views VIEW -default default test_binary_table BASE TABLE -default default test_binary_table$history BASE TABLE -default default test_binary_table$manifests BASE TABLE -default default test_binary_table$snapshots BASE TABLE -default default test_encrypted_round_trip BASE TABLE -default default test_encrypted_round_trip$history BASE TABLE -default default test_encrypted_round_trip$manifests BASE TABLE -default default test_encrypted_round_trip$snapshots BASE TABLE -default default test_partitioned_table BASE TABLE -default default test_partitioned_table$history BASE TABLE -default default test_partitioned_table$manifests BASE TABLE -default default test_partitioned_table$snapshots BASE TABLE -default information_schema columns VIEW -default information_schema df_settings VIEW -default information_schema parameters VIEW -default information_schema routines VIEW -default information_schema schemata VIEW -default information_schema tables VIEW -default information_schema views VIEW diff --git a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt deleted file mode 100644 index ffa74173dc..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt +++ /dev/null @@ -1,183 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test timestamp predicate pushdown behavior -# -# When you CREATE TABLE with TIMESTAMP in DataFusion, it creates a TimestampNs column -# (nanosecond precision) in Iceberg, since DataFusion's default TIMESTAMP type is nanoseconds. -# -# We use Datum::timestamp_nanos() for nanosecond timestamp predicates to preserve -# full precision. This allows predicates to be correctly pushed down to Iceberg. - -# Create test table with timestamp column -statement ok -CREATE TABLE default.default.test_timestamp_table (id INT NOT NULL, ts TIMESTAMP) - -# Insert test data with timestamps -# We use CAST to convert string timestamps to proper timestamp values -query I -INSERT INTO default.default.test_timestamp_table -VALUES - (1, CAST('2023-01-01 00:00:00' AS TIMESTAMP)), - (2, CAST('2023-01-05 12:30:00' AS TIMESTAMP)), - (3, CAST('2023-01-10 15:45:30' AS TIMESTAMP)), - (4, CAST('2023-01-15 09:00:00' AS TIMESTAMP)), - (5, CAST('2023-01-20 18:20:10' AS TIMESTAMP)) ----- -5 - -# Verify timestamp equality predicate IS pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_timestamp_table WHERE ts = CAST('2023-01-05 12:30:00' AS TIMESTAMP) ----- -logical_plan -01)Filter: default.default.test_timestamp_table.ts = TimestampNanosecond(1672921800000000000, None) -02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts = TimestampNanosecond(1672921800000000000, None)] -physical_plan -01)FilterExec: ts@1 = 1672921800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,ts] predicate:[ts = 2023-01-05 12:30:00] - -# Verify timestamp equality filtering works -query I? -SELECT * FROM default.default.test_timestamp_table WHERE ts = CAST('2023-01-05 12:30:00' AS TIMESTAMP) ----- -2 2023-01-05T12:30:00 - -# Verify timestamp greater than predicate IS pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_timestamp_table WHERE ts > CAST('2023-01-10 00:00:00' AS TIMESTAMP) ----- -logical_plan -01)Filter: default.default.test_timestamp_table.ts > TimestampNanosecond(1673308800000000000, None) -02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts > TimestampNanosecond(1673308800000000000, None)] -physical_plan -01)FilterExec: ts@1 > 1673308800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-10 00:00:00] - -# Verify timestamp greater than filtering -query I? rowsort -SELECT * FROM default.default.test_timestamp_table WHERE ts > CAST('2023-01-10 00:00:00' AS TIMESTAMP) ----- -3 2023-01-10T15:45:30 -4 2023-01-15T09:00:00 -5 2023-01-20T18:20:10 - -# Test timestamp less than or equal filtering -query I? rowsort -SELECT * FROM default.default.test_timestamp_table WHERE ts <= CAST('2023-01-05 12:30:00' AS TIMESTAMP) ----- -1 2023-01-01T00:00:00 -2 2023-01-05T12:30:00 - -# Verify timestamp range predicate (AND of two comparisons) IS pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_timestamp_table -WHERE ts >= CAST('2023-01-05 00:00:00' AS TIMESTAMP) - AND ts <= CAST('2023-01-15 23:59:59' AS TIMESTAMP) ----- -logical_plan -01)Filter: default.default.test_timestamp_table.ts >= TimestampNanosecond(1672876800000000000, None) AND default.default.test_timestamp_table.ts <= TimestampNanosecond(1673827199000000000, None) -02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts >= TimestampNanosecond(1672876800000000000, None), default.default.test_timestamp_table.ts <= TimestampNanosecond(1673827199000000000, None)] -physical_plan -01)FilterExec: ts@1 >= 1672876800000000000 AND ts@1 <= 1673827199000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,ts] predicate:[(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59)] - -# Test timestamp range predicate filtering -query I? rowsort -SELECT * FROM default.default.test_timestamp_table -WHERE ts >= CAST('2023-01-05 00:00:00' AS TIMESTAMP) - AND ts <= CAST('2023-01-15 23:59:59' AS TIMESTAMP) ----- -2 2023-01-05T12:30:00 -3 2023-01-10T15:45:30 -4 2023-01-15T09:00:00 - -# Test timestamp predicate combined with other predicates -query I? rowsort -SELECT * FROM default.default.test_timestamp_table -WHERE ts >= CAST('2023-01-10 00:00:00' AS TIMESTAMP) AND id < 5 ----- -3 2023-01-10T15:45:30 -4 2023-01-15T09:00:00 - -# Test timestamp NOT EQUAL predicate -query I? rowsort -SELECT * FROM default.default.test_timestamp_table WHERE ts != CAST('2023-01-05 12:30:00' AS TIMESTAMP) ----- -1 2023-01-01T00:00:00 -3 2023-01-10T15:45:30 -4 2023-01-15T09:00:00 -5 2023-01-20T18:20:10 - -# Test timestamp less than filtering -query I? rowsort -SELECT * FROM default.default.test_timestamp_table WHERE ts < CAST('2023-01-05 00:00:00' AS TIMESTAMP) ----- -1 2023-01-01T00:00:00 - -# Clean up: Drop the test table -statement ok -DROP TABLE default.default.test_timestamp_table - -# ============================================================================ -# Test timestamp predicate pushdown with different precisions -# ============================================================================ - -# Test with TIMESTAMP(6) - microsecond precision -statement ok -CREATE TABLE default.default.test_timestamp_micros (id INT NOT NULL, ts TIMESTAMP(6)) - -query I -INSERT INTO default.default.test_timestamp_micros -VALUES - (1, CAST('2023-01-01 00:00:00' AS TIMESTAMP)), - (2, CAST('2023-01-05 12:30:00' AS TIMESTAMP)) ----- -2 - -# Verify microsecond timestamp predicate is pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_timestamp_micros WHERE ts > CAST('2023-01-01 00:00:00' AS TIMESTAMP) ----- -logical_plan -01)Filter: default.default.test_timestamp_micros.ts > TimestampMicrosecond(1672531200000000, None) -02)--TableScan: default.default.test_timestamp_micros projection=[id, ts], partial_filters=[default.default.test_timestamp_micros.ts > TimestampMicrosecond(1672531200000000, None)] -physical_plan -01)FilterExec: ts@1 > 1672531200000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-01 00:00:00] - -query I? -SELECT * FROM default.default.test_timestamp_micros WHERE ts > CAST('2023-01-01 00:00:00' AS TIMESTAMP) ----- -2 2023-01-05T12:30:00 - -statement ok -DROP TABLE default.default.test_timestamp_micros - -# Test with TIMESTAMP(3) - millisecond precision -# This should fail because Iceberg doesn't support millisecond precision -statement error DataFusion error: External error: DataInvalid => Unsupported Arrow data type: Timestamp\(ms\) -CREATE TABLE default.default.test_timestamp_millis (id INT NOT NULL, ts TIMESTAMP(3)) - -# Test with TIMESTAMP(0) - second precision -# This should fail because Iceberg doesn't support second precision -statement error DataFusion error: External error: DataInvalid => Unsupported Arrow data type: Timestamp\(s\) -CREATE TABLE default.default.test_timestamp_seconds (id INT NOT NULL, ts TIMESTAMP(0)) diff --git a/crates/sqllogictest/tests/sqllogictests.rs b/crates/sqllogictest/tests/sqllogictests.rs deleted file mode 100644 index 434d9d78cc..0000000000 --- a/crates/sqllogictest/tests/sqllogictests.rs +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fs; -use std::path::PathBuf; - -use iceberg_sqllogictest::schedule::Schedule; -use libtest_mimic::{Arguments, Trial}; -use tokio::runtime::Handle; - -pub fn main() { - env_logger::init(); - - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap(); - - // Parse command line arguments - let args = Arguments::from_args(); - - let tests = collect_trials(rt.handle().clone()).unwrap(); - let result = libtest_mimic::run(&args, tests); - - drop(rt); - - result.exit(); -} - -pub(crate) fn collect_trials(handle: Handle) -> anyhow::Result> { - let schedule_files = collect_schedule_files()?; - log::debug!( - "Found {} schedules files: {:?}", - schedule_files.len(), - &schedule_files - ); - let mut trials = Vec::with_capacity(schedule_files.len()); - for schedule_file in schedule_files { - let h = handle.clone(); - let trial_name = format!( - "schedule: {}", - schedule_file - .file_name() - .expect("Schedule file should have a name") - .to_string_lossy() - ); - let trial = Trial::test(trial_name, move || { - Ok(h.block_on(run_schedule(schedule_file.clone()))?) - }); - trials.push(trial); - } - Ok(trials) -} - -pub(crate) fn collect_schedule_files() -> anyhow::Result> { - let dir = PathBuf::from(format!("{}/testdata/schedules", env!("CARGO_MANIFEST_DIR"))); - let mut schedule_files = Vec::with_capacity(32); - for entry in fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if path.is_file() { - schedule_files.push(fs::canonicalize(dir.join(path))?); - } - } - Ok(schedule_files) -} - -pub(crate) async fn run_schedule(schedule_file: PathBuf) -> anyhow::Result<()> { - let schedules = Schedule::from_file(schedule_file).await?; - schedules.run().await?; - - Ok(()) -} From 5d2c755d663e5694a0058bf40fcf2c7b662562d6 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 17 Sep 2026 23:05:34 +0000 Subject: [PATCH 03/63] fix: update rustls to 0.23.45 (#3244) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f16f025bb0..64c5c61875 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -577,9 +577,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -587,9 +587,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -5265,9 +5265,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "once_cell", @@ -5329,9 +5329,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", From 492234eb43b1aceee1b9379d5c676e4635b5d4d7 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 17 Sep 2026 23:32:03 +0000 Subject: [PATCH 04/63] ci: run security audit for merge queue (#3245) * ci: make merge queue checks reliable Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * ci: keep website checks optional Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/audit.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index c53d4af475..e02fde413f 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -30,10 +30,7 @@ on: - "**/Cargo.lock" pull_request: - paths: - - "**/Cargo.toml" - - "**/Cargo.lock" - - ".github/workflows/audit.yml" + merge_group: schedule: - cron: '0 0 * * *' From 832a4eba7a732baebbbd7f76bb6fd6225f1db1e9 Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Fri, 18 Sep 2026 13:10:20 +0000 Subject: [PATCH 05/63] ci: fix Rust cache action casing for ASF allowlist (#3249) --- .github/workflows/bindings_python_ci.yml | 2 +- .github/workflows/ci.yml | 8 ++++---- .github/workflows/public-api.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 84ce59fca8..68586abff1 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -89,7 +89,7 @@ jobs: uses: $/.github/actions/setup-builder - name: Cache Rust artifacts if: runner.os != 'Linux' - uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: key: bindings-python save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9fce8c20c..571b46fcc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,7 +137,7 @@ jobs: uses: $/.github/actions/setup-builder - name: Cache Rust artifacts - uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} @@ -158,7 +158,7 @@ jobs: uses: $/.github/actions/setup-builder - name: Cache Rust artifacts - uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} @@ -199,7 +199,7 @@ jobs: uses: $/.github/actions/setup-builder - name: Cache Rust artifacts - uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} @@ -228,7 +228,7 @@ jobs: uses: $/.github/actions/setup-builder - name: Cache Rust artifacts - uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: key: ${{ matrix.test-suite.name }} save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/public-api.yml b/.github/workflows/public-api.yml index 9b19358fd0..b65b9cf404 100644 --- a/.github/workflows/public-api.yml +++ b/.github/workflows/public-api.yml @@ -45,7 +45,7 @@ jobs: uses: $/.github/actions/setup-builder - name: Cache Rust artifacts - uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} From 53eccc5f0892eb27b4af940e5e10993dc6ab9e4a Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 21 Sep 2026 20:18:28 +0000 Subject: [PATCH 06/63] fix(cache): drop schema-id from the manifest-list cache key (#3235) * fix(cache): drop schema-id from the manifest-list cache key ObjectCache::get_manifest_list built the cache key from snapshot.schema_id().unwrap(), but schema-id is optional on snapshots in every table version (v1-v3 per spec, and already read as Option), so a snapshot that omits it panicked on every scan (the object cache is on by default). schema-id does not belong in this key: ManifestListReader::load parses the manifest list from its bytes and the table format version only and never reads schema-id, and the manifest-list location is already unique per snapshot. * Fix import * Harden test --- crates/iceberg/src/io/object_cache.rs | 54 +++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index d780e60c1f..01a556b9cb 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -22,7 +22,7 @@ use crate::encryption::EncryptionManager; use crate::io::FileIO; use crate::spec::{ FormatVersion, Manifest, ManifestFile, ManifestList, ManifestListReader, ManifestReader, - SchemaId, SnapshotRef, TableMetadataRef, + SnapshotRef, TableMetadataRef, }; use crate::{Error, ErrorKind, Result}; @@ -36,7 +36,7 @@ pub(crate) enum CachedItem { #[derive(Clone, Debug, Hash, Eq, PartialEq)] pub(crate) enum CachedObjectKey { - ManifestList((String, FormatVersion, SchemaId)), + ManifestList((String, FormatVersion)), // The manifest-level `first_row_id` is part of the key because the parsed // manifest inherits it onto its entries: the same physical manifest can be // referenced with different offsets across snapshots and branches, so it @@ -159,7 +159,6 @@ impl ObjectCache { let key = CachedObjectKey::ManifestList(( snapshot.manifest_list().to_string(), table_metadata.format_version, - snapshot.schema_id().unwrap(), )); let cache_entry = self .cache @@ -214,6 +213,7 @@ impl ObjectCache { #[cfg(test)] mod tests { + use std::collections::HashMap; use std::fs; use minijinja::value::Value; @@ -226,7 +226,8 @@ mod tests { use crate::io::{FileIO, OutputFile}; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Literal, ManifestEntry, - ManifestListWriter, ManifestStatus, ManifestWriterBuilder, Struct, TableMetadata, + ManifestListWriter, ManifestStatus, ManifestWriterBuilder, Operation, Snapshot, Struct, + Summary, TableMetadata, }; use crate::table::Table; use crate::test_utils::test_runtime; @@ -444,6 +445,51 @@ mod tests { ); } + #[tokio::test] + async fn test_get_manifest_list_with_no_schema_id() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + let current_snapshot = fixture.table.metadata().current_snapshot().unwrap(); + + // The spec marks `schema-id` optional in every version (v1-v3), so a + // snapshot may omit it; fetching its manifest list must not depend on the + // schema-id being present. + let snapshot_without_schema_id: SnapshotRef = Snapshot::builder() + .with_snapshot_id(current_snapshot.snapshot_id()) + .with_sequence_number(current_snapshot.sequence_number()) + .with_timestamp_ms(current_snapshot.timestamp_ms()) + .with_manifest_list(current_snapshot.manifest_list()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build() + .into(); + assert!(snapshot_without_schema_id.schema_id().is_none()); + assert!(current_snapshot.schema_id().is_some()); + + let object_cache = ObjectCache::new(fixture.table.file_io().clone(), None); + + // Cold miss: the schema-id-less snapshot populates the cache. + let inserted = object_cache + .get_manifest_list(&snapshot_without_schema_id, &fixture.table.metadata_ref()) + .await + .unwrap(); + assert_eq!(inserted.entries().len(), 1); + + // Warm hit: the original snapshot carries a schema-id but points at the same + // manifest-list location, so it returns the same cached entry. + let cached = object_cache + .get_manifest_list(current_snapshot, &fixture.table.metadata_ref()) + .await + .unwrap(); + assert!( + Arc::ptr_eq(&inserted, &cached), + "snapshots with and without schema-id at one location must share a cache entry" + ); + } + #[tokio::test] async fn test_get_manifest_keys_on_first_row_id() { use crate::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Type}; From f3e41d5321d0043697041f1353eb47d46f2b4b1d Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 21 Sep 2026 21:26:11 +0000 Subject: [PATCH 07/63] feat(transaction): support transform-based sort orders (#2765) * feat(transaction): support transform-based sort orders ReplaceSortOrderAction only supported sorting by a column's raw value: `asc`/`desc` hardcoded Transform::Identity and only accepted a column name. Java's SortOrderBuilder.asc/desc accept a Term, which can be a transform expression (bucket[N], year, truncate[W], ...). Add asc_with_transform / desc_with_transform. asc/desc are unchanged (same signature and behavior) and now delegate to the new methods with Transform::Identity, so no existing caller is affected. Transform-compatibility with the source column's type is checked at commit time via the existing SortOrder::builder().build(schema) -> check_compatibility path, matching the timing of Java's SortOrder.Builder.build() -- the spec-level type was already general (SortField.transform, check_compatibility already validate arbitrary transforms); only the transaction-layer API was narrow. Scope: this only extends the metadata-declaration API. Whether the write path sorts data to match a table's declared sort order -- for any transform, including the pre-existing identity case -- is a separate, pre-existing gap untouched here. Closes #2764 * test(transaction): metadata.json round-trip test for transform sort orders Per review: commit a transform-based sort order through the in-crate memory catalog (which serializes the updated metadata to a metadata.json file via TableMetadata::write_to), then reload the table (TableMetadata::read_from parses that file) and assert the transforms survive the JSON round-trip. * docs(transaction): note intended Term convergence; hoist test imports Per review: document on asc_with_transform/desc_with_transform that once Term becomes transform-carrying (#2665), sort-order declaration is expected to converge on Term-based asc/desc (as in Java's SortOrderBuilder) and the _with_transform variants can be deprecated. Also hoist the use statements scattered inside test fns to the test module top for consistency. * fix(transaction): validate sort transforms and preserve input errors * chore: update public API snapshot for sort transforms * fix(sort): use consistent validation errors and document void policy --- crates/iceberg/public-api.txt | 2 + crates/iceberg/src/spec/sort.rs | 47 ++--- crates/iceberg/src/transaction/sort_order.rs | 196 ++++++++++++++++++- 3 files changed, 210 insertions(+), 35 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index a917d28f5a..f29c5efe07 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3254,7 +3254,9 @@ pub fn iceberg::transaction::FastAppendAction::with_check_duplicate(self, v: boo pub struct iceberg::transaction::ReplaceSortOrderAction impl iceberg::transaction::ReplaceSortOrderAction pub fn iceberg::transaction::ReplaceSortOrderAction::asc(self, name: &str, null_order: iceberg::spec::NullOrder) -> Self +pub fn iceberg::transaction::ReplaceSortOrderAction::asc_with_transform(self, name: &str, transform: iceberg::spec::Transform, null_order: iceberg::spec::NullOrder) -> Self pub fn iceberg::transaction::ReplaceSortOrderAction::desc(self, name: &str, null_order: iceberg::spec::NullOrder) -> Self +pub fn iceberg::transaction::ReplaceSortOrderAction::desc_with_transform(self, name: &str, transform: iceberg::spec::Transform, null_order: iceberg::spec::NullOrder) -> Self pub struct iceberg::transaction::Transaction impl iceberg::transaction::Transaction pub async fn iceberg::transaction::Transaction::commit(self, catalog: &dyn iceberg::Catalog) -> iceberg::Result diff --git a/crates/iceberg/src/spec/sort.rs b/crates/iceberg/src/spec/sort.rs index b27d5a36d3..e40aaa65dd 100644 --- a/crates/iceberg/src/spec/sort.rs +++ b/crates/iceberg/src/spec/sort.rs @@ -180,7 +180,7 @@ impl SortOrderBuilder { match schema.field_by_id(sort_field.source_id) { None => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::DataInvalid, format!("Cannot find source column for sort field: {sort_field}"), )); } @@ -189,20 +189,13 @@ impl SortOrderBuilder { if !source_type.is_primitive() { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::DataInvalid, format!("Cannot sort by non-primitive source field: {source_type}"), )); } let field_transform = sort_field.transform; - if field_transform.result_type(source_type).is_err() { - return Err(Error::new( - ErrorKind::Unexpected, - format!( - "Invalid source type {source_type} for transform {field_transform}" - ), - )); - } + field_transform.result_type(source_type)?; } } } @@ -403,12 +396,12 @@ mod tests { ) .build(&schema); + let err = sort_order_builder_result.expect_err("Expected an Err value"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); assert_eq!( - sort_order_builder_result - .expect_err("Expected an Err value") - .message(), + err.message(), "Cannot find source column for sort field: SortField { source_id: 2, transform: identity, direction: ascending, null_order: first }" - ) + ); } #[test] @@ -444,12 +437,12 @@ mod tests { ) .build(&schema); + let err = sort_order_builder_result.expect_err("Expected an Err value"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); assert_eq!( - sort_order_builder_result - .expect_err("Expected an Err value") - .message(), + err.message(), "Cannot sort by non-primitive source field: list" - ) + ); } #[test] @@ -473,12 +466,12 @@ mod tests { ) .build(&schema); + let err = sort_order_builder_result.expect_err("Expected an Err value"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); assert_eq!( - sort_order_builder_result - .expect_err("Expected an Err value") - .message(), + err.message(), "Cannot sort by non-primitive source field: variant" - ) + ); } #[test] @@ -502,12 +495,12 @@ mod tests { ) .build(&schema); + let err = sort_order_builder_result.expect_err("Expected an Err value"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); assert_eq!( - sort_order_builder_result - .expect_err("Expected an Err value") - .message(), - "Invalid source type int for transform year" - ) + err.message(), + "int is not a valid input type of year transform" + ); } #[test] diff --git a/crates/iceberg/src/transaction/sort_order.rs b/crates/iceberg/src/transaction/sort_order.rs index 964bbccd88..17655fe235 100644 --- a/crates/iceberg/src/transaction/sort_order.rs +++ b/crates/iceberg/src/transaction/sort_order.rs @@ -31,6 +31,7 @@ use crate::{Error, ErrorKind, TableRequirement, TableUpdate}; #[derive(Debug, PartialEq, Eq, Clone)] struct PendingSortField { name: String, + transform: Transform, direction: SortDirection, null_order: NullOrder, } @@ -44,9 +45,18 @@ impl PendingSortField { ) })?; + // This action intentionally refuses to author all-null (void) sort keys; + // this policy does not restrict reading or preserving existing metadata. + if matches!(self.transform, Transform::Unknown | Transform::Void) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Cannot sort by transform {}", self.transform), + )); + } + Ok(SortField::builder() .source_id(field_id) - .transform(Transform::Identity) + .transform(self.transform) .direction(self.direction) .null_order(self.null_order) .build()) @@ -65,24 +75,64 @@ impl ReplaceSortOrderAction { } } - /// Adds a field for sorting in ascending order. + /// Adds a field for sorting in ascending order, sorting by the column's raw value + /// (an identity transform). To sort by a transform of the column instead (e.g. + /// `bucket[N]`, `year`, `truncate[W]`), use [`Self::asc_with_transform`]. pub fn asc(self, name: &str, null_order: NullOrder) -> Self { - self.add_sort_field(name, SortDirection::Ascending, null_order) + self.asc_with_transform(name, Transform::Identity, null_order) } - /// Adds a field for sorting in descending order. + /// Adds a field for sorting in descending order, sorting by the column's raw value + /// (an identity transform). To sort by a transform of the column instead (e.g. + /// `bucket[N]`, `year`, `truncate[W]`), use [`Self::desc_with_transform`]. pub fn desc(self, name: &str, null_order: NullOrder) -> Self { - self.add_sort_field(name, SortDirection::Descending, null_order) + self.desc_with_transform(name, Transform::Identity, null_order) + } + + /// Adds a field for sorting in ascending order by a transform of the column's value + /// (e.g. `Transform::Bucket(16)`, `Transform::Year`, `Transform::Truncate(4)`). + /// + /// Whether the transform is valid for the column's type is checked at commit time, + /// once the table schema is available (mirroring Java's `SortOrder.Builder.build()`). + /// `Transform::Unknown` and `Transform::Void` are rejected at commit time with + /// [`ErrorKind::DataInvalid`]. Rejecting `Void` is an intentional authoring policy + /// for this action: an all-null sort key adds no ordering information. + /// + /// Note: `Term` is currently a plain column reference. Once it becomes + /// transform-carrying (#2665), sort-order declaration is expected to converge on + /// Term-based `asc`/`desc` (as in Java's `SortOrderBuilder`), at which point the + /// `_with_transform` variants can be deprecated in its favor. + pub fn asc_with_transform( + self, + name: &str, + transform: Transform, + null_order: NullOrder, + ) -> Self { + self.add_sort_field(name, transform, SortDirection::Ascending, null_order) + } + + /// Adds a field for sorting in descending order by a transform of the column's value. + /// + /// See [`Self::asc_with_transform`] for validation and future API direction. + pub fn desc_with_transform( + self, + name: &str, + transform: Transform, + null_order: NullOrder, + ) -> Self { + self.add_sort_field(name, transform, SortDirection::Descending, null_order) } fn add_sort_field( mut self, name: &str, + transform: Transform, sort_direction: SortDirection, null_order: NullOrder, ) -> Self { self.pending_sort_fields.push(PendingSortField { name: name.to_string(), + transform, direction: sort_direction, null_order, }); @@ -127,12 +177,17 @@ impl TransactionAction for ReplaceSortOrderAction { #[cfg(test)] mod tests { + use std::sync::Arc; + use as_any::Downcast; - use crate::spec::{NullOrder, SortDirection}; + use crate::catalog::Catalog; + use crate::memory::tests::new_memory_catalog; + use crate::spec::{NullOrder, SortDirection, Transform}; use crate::transaction::sort_order::{PendingSortField, ReplaceSortOrderAction}; - use crate::transaction::tests::make_v2_table; - use crate::transaction::{ApplyTransactionAction, Transaction}; + use crate::transaction::tests::{make_v2_table, make_v3_minimal_table_in_catalog}; + use crate::transaction::{ApplyTransactionAction, Transaction, TransactionAction}; + use crate::{ErrorKind, TableUpdate}; #[test] fn test_replace_sort_order() { @@ -153,14 +208,139 @@ mod tests { assert_eq!(replace_sort_order.pending_sort_fields, vec![ PendingSortField { name: String::from("x"), + transform: Transform::Identity, direction: SortDirection::Ascending, null_order: NullOrder::First, }, PendingSortField { name: String::from("y"), + transform: Transform::Identity, direction: SortDirection::Descending, null_order: NullOrder::Last, } ]); } + + #[test] + fn test_replace_sort_order_with_transform() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + let replace_sort_order = tx.replace_sort_order(); + + let tx = replace_sort_order + .asc_with_transform("x", Transform::Bucket(16), NullOrder::First) + .desc_with_transform("y", Transform::Truncate(4), NullOrder::Last) + .apply(tx) + .unwrap(); + + let replace_sort_order = (*tx.actions[0]) + .downcast_ref::() + .unwrap(); + + assert_eq!(replace_sort_order.pending_sort_fields, vec![ + PendingSortField { + name: String::from("x"), + transform: Transform::Bucket(16), + direction: SortDirection::Ascending, + null_order: NullOrder::First, + }, + PendingSortField { + name: String::from("y"), + transform: Transform::Truncate(4), + direction: SortDirection::Descending, + null_order: NullOrder::Last, + } + ]); + } + + #[tokio::test] + async fn test_replace_sort_order_with_transform_commits() { + let table = make_v2_table(); + let action = Arc::new(ReplaceSortOrderAction::new().asc_with_transform( + "x", + Transform::Bucket(16), + NullOrder::First, + )); + + let mut action_commit = TransactionAction::commit(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + assert_eq!(updates.len(), 2); + let TableUpdate::AddSortOrder { sort_order } = &updates[0] else { + panic!("expected AddSortOrder, got {:?}", updates[0]); + }; + assert_eq!(sort_order.fields.len(), 1); + assert_eq!(sort_order.fields[0].transform, Transform::Bucket(16)); + } + + #[tokio::test] + async fn test_replace_sort_order_rejects_incompatible_transform() { + let table = make_v2_table(); + // `x` is a `long` column; `year` only accepts date/timestamp types. + let action = Arc::new(ReplaceSortOrderAction::new().asc_with_transform( + "x", + Transform::Year, + NullOrder::First, + )); + + let err = TransactionAction::commit(action, &table) + .await + .err() + .expect("year transform on a long column should be rejected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_replace_sort_order_rejects_unknown_and_void_transforms() { + let table = make_v2_table(); + for transform in [Transform::Unknown, Transform::Void] { + for action in [ + ReplaceSortOrderAction::new().asc_with_transform("x", transform, NullOrder::First), + ReplaceSortOrderAction::new().desc_with_transform("x", transform, NullOrder::Last), + ] { + let err = TransactionAction::commit(Arc::new(action), &table) + .await + .err() + .expect("unknown and void sort transforms should be rejected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + format!("Cannot sort by transform {transform}") + ); + } + } + } + + #[tokio::test] + async fn test_sort_order_transform_survives_metadata_json_round_trip() { + // Commit a transform-based sort order through a real catalog: the memory + // catalog serializes the updated table metadata to a metadata.json file + // (`TableMetadata::write_to`), and `load_table` reads that file back and + // parses it (`TableMetadata::read_from`). This exercises the full + // JSON round-trip of the transform (e.g. `"bucket[16]"`), not just the + // in-memory `TableUpdate`. + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let tx = Transaction::new(&table); + let tx = tx + .replace_sort_order() + .asc_with_transform("x", Transform::Bucket(16), NullOrder::First) + .desc_with_transform("y", Transform::Truncate(4), NullOrder::Last) + .apply(tx) + .unwrap(); + let committed = tx.commit(&catalog).await.unwrap(); + + // Reload from the catalog: this parses the metadata.json written above. + let reloaded = catalog.load_table(committed.identifier()).await.unwrap(); + let sort_order = reloaded.metadata().default_sort_order(); + + assert_eq!(sort_order.fields.len(), 2); + assert_eq!(sort_order.fields[0].transform, Transform::Bucket(16)); + assert_eq!(sort_order.fields[0].direction, SortDirection::Ascending); + assert_eq!(sort_order.fields[0].null_order, NullOrder::First); + assert_eq!(sort_order.fields[1].transform, Transform::Truncate(4)); + assert_eq!(sort_order.fields[1].direction, SortDirection::Descending); + assert_eq!(sort_order.fields[1].null_order, NullOrder::Last); + } } From bb1e4a4861f02377489eff818b75138f414c4cb0 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 21 Sep 2026 21:42:58 +0000 Subject: [PATCH 08/63] perf(expr): resolve row group stats once per eq/in/starts_with leaf (#3254) The eq, in, and starts_with row-group metric checks each read both the lower and upper bound via min_value and max_value, and each of those independently re-runs stats_and_type_for_field_id. Resolve the stats and type once per leaf and decode each bound in place, so the lower and upper bounds still decode lazily. --- .../visitors/row_group_metrics_evaluator.rs | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs index 3a5a406e78..ab8d81ce26 100644 --- a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs @@ -297,7 +297,11 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { return ROW_GROUP_CANT_MATCH; } - if let Some(lower_bound) = self.min_value(field_id)? { + let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { + return ROW_GROUP_MIGHT_MATCH; + }; + + if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { if lower_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -307,7 +311,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = self.max_value(field_id)? { + if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { if upper_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -351,7 +355,11 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { )); }; - if let Some(lower_bound) = self.min_value(field_id)? { + let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { + return ROW_GROUP_MIGHT_MATCH; + }; + + if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { let PrimitiveLiteral::String(lower_bound) = lower_bound.literal() else { return Err(Error::new( ErrorKind::Unexpected, @@ -369,7 +377,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = self.max_value(field_id)? { + if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { let PrimitiveLiteral::String(upper_bound) = upper_bound.literal() else { return Err(Error::new( ErrorKind::Unexpected, @@ -476,7 +484,11 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { return ROW_GROUP_MIGHT_MATCH; } - if let Some(lower_bound) = self.min_value(field_id)? { + let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { + return ROW_GROUP_MIGHT_MATCH; + }; + + if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { if lower_bound.is_nan() { // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. return ROW_GROUP_MIGHT_MATCH; @@ -488,7 +500,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = self.max_value(field_id)? { + if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { if upper_bound.is_nan() { // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. return ROW_GROUP_MIGHT_MATCH; From e694eea42032a3f0898ccee33f5d3dd0ca9cd24b Mon Sep 17 00:00:00 2001 From: Xander Date: Tue, 22 Sep 2026 19:40:50 +0000 Subject: [PATCH 09/63] Introduce invalid data error macro (#2928) * Introduce invalid data error macro * fix * comments * fix * fix --------- Co-authored-by: blackmwk <257669749+blackmwk@users.noreply.github.com> --- .../src/arrow/caching_delete_file_loader.rs | 78 ++---- .../src/arrow/partition_value_calculator.rs | 20 +- crates/iceberg/src/arrow/reader/pipeline.rs | 11 +- .../src/arrow/reader/predicate_visitor.rs | 19 +- crates/iceberg/src/arrow/reader/projection.rs | 9 +- .../iceberg/src/arrow/reader/row_lineage.rs | 7 +- .../arrow/record_batch_partition_splitter.rs | 27 +- .../src/arrow/record_batch_projector.rs | 13 +- .../src/arrow/record_batch_transformer.rs | 32 +-- crates/iceberg/src/arrow/schema.rs | 118 +++----- crates/iceberg/src/arrow/value.rs | 262 +++++------------- crates/iceberg/src/avro/schema.rs | 53 ++-- crates/iceberg/src/catalog/memory/catalog.rs | 11 +- .../src/catalog/memory/namespace_state.rs | 6 +- crates/iceberg/src/catalog/mod.rs | 12 +- crates/iceberg/src/delete_file_index.rs | 61 ++-- crates/iceberg/src/delete_vector.rs | 65 ++--- crates/iceberg/src/encryption/crypto.rs | 32 +-- crates/iceberg/src/encryption/key_metadata.rs | 26 +- crates/iceberg/src/encryption/kms/memory.rs | 16 +- crates/iceberg/src/encryption/manager.rs | 44 +-- crates/iceberg/src/encryption/stream.rs | 72 ++--- crates/iceberg/src/error.rs | 71 +++++ crates/iceberg/src/expr/predicate.rs | 11 +- crates/iceberg/src/expr/term.rs | 21 +- .../expr/visitors/strict_metrics_evaluator.rs | 8 +- crates/iceberg/src/inspect/manifests.rs | 13 +- crates/iceberg/src/io/storage/config/s3.rs | 10 +- crates/iceberg/src/io/storage/local_fs.rs | 45 +-- crates/iceberg/src/io/storage/memory.rs | 36 +-- crates/iceberg/src/partitioning.rs | 20 +- crates/iceberg/src/puffin/metadata.rs | 56 ++-- crates/iceberg/src/puffin/mod.rs | 18 +- crates/iceberg/src/scan/mod.rs | 13 +- crates/iceberg/src/scan/task.rs | 32 +-- crates/iceberg/src/spec/manifest/_serde.rs | 15 +- crates/iceberg/src/spec/manifest/data_file.rs | 14 +- crates/iceberg/src/spec/manifest/entry.rs | 9 +- crates/iceberg/src/spec/manifest/metadata.rs | 45 +-- crates/iceberg/src/spec/manifest/mod.rs | 21 +- crates/iceberg/src/spec/manifest/reader.rs | 18 +- crates/iceberg/src/spec/manifest/writer.rs | 44 +-- .../iceberg/src/spec/manifest_list/_serde.rs | 68 +---- .../src/spec/manifest_list/manifest_file.rs | 14 +- .../iceberg/src/spec/manifest_list/writer.rs | 41 +-- crates/iceberg/src/spec/name_mapping/mod.rs | 9 +- crates/iceberg/src/spec/partition.rs | 98 +++---- .../iceberg/src/spec/schema/id_reassigner.rs | 28 +- crates/iceberg/src/spec/schema/index.rs | 8 +- crates/iceberg/src/spec/schema/mod.rs | 23 +- .../iceberg/src/spec/schema/prune_columns.rs | 25 +- crates/iceberg/src/spec/schema/utils.rs | 8 +- crates/iceberg/src/spec/snapshot.rs | 23 +- crates/iceberg/src/spec/snapshot_summary.rs | 6 +- crates/iceberg/src/spec/table_metadata.rs | 190 +++++-------- .../src/spec/table_metadata_builder.rs | 242 ++++++---------- crates/iceberg/src/spec/table_properties.rs | 47 ++-- crates/iceberg/src/spec/transform.rs | 97 +++---- crates/iceberg/src/spec/values/datum.rs | 122 +++----- .../iceberg/src/spec/values/decimal_utils.rs | 6 +- crates/iceberg/src/spec/values/literal.rs | 185 +++++-------- crates/iceberg/src/spec/values/serde.rs | 49 +--- crates/iceberg/src/spec/view_metadata.rs | 17 +- .../iceberg/src/spec/view_metadata_builder.rs | 76 ++--- crates/iceberg/src/spec/view_version.rs | 10 +- crates/iceberg/src/table.rs | 33 +-- .../src/transaction/expire_snapshots.rs | 20 +- crates/iceberg/src/transaction/snapshot.rs | 31 +-- crates/iceberg/src/transaction/sort_order.rs | 23 +- .../src/transaction/update_location.rs | 8 +- .../src/transaction/upgrade_format_version.rs | 8 +- crates/iceberg/src/transform/temporal.rs | 26 +- crates/iceberg/src/transform/truncate.rs | 11 +- .../writer/base_writer/data_file_writer.rs | 9 +- .../base_writer/equality_delete_writer.rs | 28 +- .../base_writer/position_delete_writer.rs | 80 ++---- .../src/writer/file_writer/parquet_writer.rs | 45 ++- 77 files changed, 1134 insertions(+), 2094 deletions(-) diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 905fa86d83..0208a678ea 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -29,6 +29,7 @@ use crate::arrow::scan_metrics::ScanMetrics; use crate::arrow::{arrow_primitive_to_literal, arrow_schema_to_schema}; use crate::delete_vector::DeleteVector; use crate::encryption::{EncryptedInputFile, StandardKeyMetadata}; +use crate::error::invalid_data; use crate::expr::Predicate::AlwaysTrue; use crate::expr::{Predicate, Reference}; use crate::io::FileIO; @@ -348,55 +349,37 @@ impl CachingDeleteFileLoader { task: &FileScanTaskDeleteFile, ) -> Result<(u64, u64, String, u64)> { let content_offset = task.content_offset.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} is missing content_offset", - task.file_path - ), + invalid_data!( + "deletion vector {} is missing content_offset", + task.file_path ) })?; let content_size = task.content_size_in_bytes.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} is missing content_size_in_bytes", - task.file_path - ), + invalid_data!( + "deletion vector {} is missing content_size_in_bytes", + task.file_path ) })?; let data_file_path = task.referenced_data_file.clone().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} is missing referenced_data_file", - task.file_path - ), + invalid_data!( + "deletion vector {} is missing referenced_data_file", + task.file_path ) })?; let record_count = task.record_count.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("deletion vector {} is missing record_count", task.file_path), - ) + invalid_data!("deletion vector {} is missing record_count", task.file_path) })?; let start = u64::try_from(content_offset).map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} has negative content_offset {content_offset}", - task.file_path - ), + invalid_data!( + "deletion vector {} has negative content_offset {content_offset}", + task.file_path ) })?; let len = u64::try_from(content_size).map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} has negative content_size_in_bytes {content_size}", - task.file_path - ), + invalid_data!( + "deletion vector {} has negative content_size_in_bytes {content_size}", + task.file_path ) })?; @@ -412,11 +395,8 @@ impl CachingDeleteFileLoader { ) -> Result<()> { let actual = delete_vector.len(); if actual != expected { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {dv_path} decoded to {actual} positions, expected {expected} from record_count" - ), + return Err(invalid_data!( + "deletion vector {dv_path} decoded to {actual} positions, expected {expected} from record_count" )); } Ok(()) @@ -528,15 +508,13 @@ impl CachingDeleteFileLoader { let columns = batch.columns(); let Some(file_paths) = columns[0].as_any().downcast_ref::() else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Could not downcast file paths array to StringArray", + return Err(invalid_data!( + "Could not downcast file paths array to StringArray" )); }; let Some(positions) = columns[1].as_any().downcast_ref::() else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Could not downcast positions array to Int64Array", + return Err(invalid_data!( + "Could not downcast positions array to Int64Array" )); }; @@ -551,15 +529,11 @@ impl CachingDeleteFileLoader { for (file_path, pos) in file_paths.iter().zip(positions.iter()) { let (Some(file_path), Some(pos)) = (file_path, pos) else { - return Err(Error::new( - ErrorKind::DataInvalid, - "null values in delete file", - )); + return Err(invalid_data!("null values in delete file")); }; if pos < 0 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("negative position in delete file {file_path}: {pos}"), + return Err(invalid_data!( + "negative position in delete file {file_path}: {pos}" )); } diff --git a/crates/iceberg/src/arrow/partition_value_calculator.rs b/crates/iceberg/src/arrow/partition_value_calculator.rs index 3520f75ac5..c8c917779a 100644 --- a/crates/iceberg/src/arrow/partition_value_calculator.rs +++ b/crates/iceberg/src/arrow/partition_value_calculator.rs @@ -27,9 +27,10 @@ use arrow_schema::DataType; use super::record_batch_projector::RecordBatchProjector; use super::type_to_arrow_type; +use crate::Result; +use crate::error::invalid_data; use crate::spec::{PartitionSpec, Schema, StructType, Type}; use crate::transform::{BoxedTransformFunction, create_transform_function}; -use crate::{Error, ErrorKind, Result}; /// Calculator for partition values in Iceberg tables. /// @@ -63,9 +64,8 @@ impl PartitionValueCalculator { /// - Projector initialization fails pub fn try_new(partition_spec: &PartitionSpec, table_schema: &Schema) -> Result { if partition_spec.is_unpartitioned() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot create partition calculator for unpartitioned table", + return Err(invalid_data!( + "Cannot create partition calculator for unpartitioned table" )); } @@ -140,10 +140,7 @@ impl PartitionValueCalculator { let expected_struct_fields = match &self.partition_arrow_type { DataType::Struct(fields) => fields.clone(), _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - "Expected partition type must be a struct", - )); + return Err(invalid_data!("Expected partition type must be a struct")); } }; @@ -156,12 +153,7 @@ impl PartitionValueCalculator { // Construct the StructArray let struct_array = StructArray::try_new(expected_struct_fields, partition_values, None) - .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to create partition struct array: {e}"), - ) - })?; + .map_err(|e| invalid_data!("Failed to create partition struct array: {e}"))?; Ok(Arc::new(struct_array)) } diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 7ec9bf4d55..ee027c5e72 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -43,7 +43,7 @@ use crate::arrow::int96::coerce_int96_timestamps; use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; use crate::arrow::scan_metrics::{CountingFileRead, ScanMetrics, ScanResult}; use crate::encryption::StandardKeyMetadata; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::expr::BoundPredicate; use crate::expr::visitors::bloom_filter_evaluator::{ BloomFilterEvaluator, ColumnBloomFilter, collect_bloom_filter_field_ids, @@ -475,12 +475,9 @@ impl FileScanTaskReader { // inheritance a committed entry always has one, so this is a malformed // manifest rather than a legitimate null. (Some(_), None) => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Data file {} has a first_row_id but no data sequence number", - task.data_file_path() - ), + return Err(invalid_data!( + "Data file {} has a first_row_id but no data sequence number", + task.data_file_path() )); } }; diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 2b05e3ebc3..264c34a9a3 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -35,11 +35,10 @@ use fnv::FnvHashSet; use parquet::schema::types::SchemaDescriptor; use crate::arrow::get_arrow_datum; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::expr::visitors::bound_predicate_visitor::BoundPredicateVisitor; use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::Datum; -use crate::{Error, ErrorKind}; /// A visitor to collect field ids from bound predicates. pub(super) struct CollectFieldIdVisitor { @@ -215,12 +214,9 @@ impl PredicateConverter<'_> { // The leaf column's index in Parquet schema. if let Some(column_idx) = self.column_map.get(&reference.field().id) { if self.parquet_schema.get_column_root(*column_idx).is_group() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Leaf column `{}` in predicates isn't a root column in Parquet schema.", - reference.field().name - ), + return Err(invalid_data!( + "Leaf column `{}` in predicates isn't a root column in Parquet schema.", + reference.field().name )); } @@ -229,13 +225,10 @@ impl PredicateConverter<'_> { .column_indices .iter() .position(|&idx| idx == *column_idx) - .ok_or(Error::new( - ErrorKind::DataInvalid, - format!( + .ok_or(invalid_data!( "Leaf column `{}` in predicates cannot be found in the required column indices.", reference.field().name - ), - ))?; + ))?; Ok(Some(index)) } else { diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index 873adb05ca..0f34f00b37 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -29,7 +29,7 @@ use parquet::schema::types::{SchemaDescriptor, Type as ParquetType}; use super::{ArrowReader, CollectFieldIdVisitor}; use crate::arrow::arrow_schema_to_schema; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::spec::{NameMapping, NestedField, PrimitiveType, Schema, Type}; @@ -299,11 +299,8 @@ pub(super) fn build_field_id_map( column_map.insert(basic_info.id(), idx); } ParquetType::GroupType { .. } => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Leaf column in schema should be primitive type but got {field_type:?}" - ), + return Err(invalid_data!( + "Leaf column in schema should be primitive type but got {field_type:?}" )); } }; diff --git a/crates/iceberg/src/arrow/reader/row_lineage.rs b/crates/iceberg/src/arrow/reader/row_lineage.rs index ee8dda4e6a..9510f71c59 100644 --- a/crates/iceberg/src/arrow/reader/row_lineage.rs +++ b/crates/iceberg/src/arrow/reader/row_lineage.rs @@ -32,6 +32,7 @@ use arrow_schema::{DataType, Field, Schema}; use arrow_select::zip::zip; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use crate::error::invalid_data; use crate::metadata_columns::{ RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, }; @@ -75,9 +76,9 @@ pub(crate) fn synthesize_row_id_column( match column_by_field_id(&batch, RESERVED_FIELD_ID_ROW_ID) { Some(id) => { if id.data_type() != &DataType::Int64 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("_row_id source must be Int64, got {}", id.data_type()), + return Err(invalid_data!( + "_row_id source must be Int64, got {}", + id.data_type() )); } diff --git a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs index 598bbf09f7..6b3aa73d22 100644 --- a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs +++ b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs @@ -24,8 +24,9 @@ use arrow_select::filter::filter_record_batch; use super::arrow_struct_to_literal; use super::partition_value_calculator::PartitionValueCalculator; +use crate::Result; +use crate::error::invalid_data; use crate::spec::{Literal, PartitionKey, PartitionSpecRef, SchemaRef, StructType}; -use crate::{Error, ErrorKind, Result}; /// Column name for the projected partition values struct pub const PROJECTED_PARTITION_VALUE_COLUMN: &str = "_partition"; @@ -129,9 +130,8 @@ impl RecordBatchPartitionSplitter { if let Some(Literal::Struct(s)) = s { Ok(s) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "Partition value is not a struct literal or is null", + Err(invalid_data!( + "Partition value is not a struct literal or is null" )) } }) @@ -141,23 +141,15 @@ impl RecordBatchPartitionSplitter { let partition_column = batch .column_by_name(PROJECTED_PARTITION_VALUE_COLUMN) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Partition column '{PROJECTED_PARTITION_VALUE_COLUMN}' not found in batch" - ), + invalid_data!( + "Partition column '{PROJECTED_PARTITION_VALUE_COLUMN}' not found in batch" ) })?; let partition_struct_array = partition_column .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Partition column is not a StructArray", - ) - })?; + .ok_or_else(|| invalid_data!("Partition column is not a StructArray"))?; let arrow_struct_array = Arc::new(partition_struct_array.clone()) as ArrayRef; let struct_array = arrow_struct_to_literal(&arrow_struct_array, &self.partition_type)?; @@ -168,9 +160,8 @@ impl RecordBatchPartitionSplitter { if let Some(Literal::Struct(s)) = s { Ok(s) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "Partition value is not a struct literal or is null", + Err(invalid_data!( + "Partition value is not a struct literal or is null" )) } }) diff --git a/crates/iceberg/src/arrow/record_batch_projector.rs b/crates/iceberg/src/arrow/record_batch_projector.rs index 7028eee961..d0802ee53b 100644 --- a/crates/iceberg/src/arrow/record_batch_projector.rs +++ b/crates/iceberg/src/arrow/record_batch_projector.rs @@ -23,7 +23,7 @@ use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use crate::arrow::schema::schema_to_arrow_schema; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::Schema as IcebergSchema; use crate::{Error, ErrorKind}; @@ -97,12 +97,9 @@ impl RecordBatchProjector { let field_id_fetch_func = |field: &Field| -> Result> { if let Some(value) = field.metadata().get(PARQUET_FIELD_ID_META_KEY) { let field_id = value.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to parse field id".to_string(), - ) - .with_context("value", value) - .with_source(e) + invalid_data!("Failed to parse field id") + .with_context("value", value) + .with_source(e) })?; Ok(Some(field_id as i64)) } else { @@ -167,7 +164,7 @@ impl RecordBatchProjector { self.projected_schema.clone(), self.project_column(batch.columns())?, ) - .map_err(|err| Error::new(ErrorKind::DataInvalid, format!("{err}"))) + .map_err(|err| invalid_data!("{err}")) } /// Do projection with columns diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index 800dc6da41..19976a81c5 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -36,6 +36,7 @@ use crate::arrow::{ datum_to_arrow_type_with_ree, primitive_type_to_arrow_type_with_ree, schema_to_arrow_schema, type_to_arrow_type, }; +use crate::error::invalid_data; use crate::metadata_columns::{ RESERVED_COL_NAME_PARTITION, RESERVED_FIELD_ID_PARTITION, get_metadata_field, }; @@ -279,13 +280,10 @@ impl StructConstant { /// the same length. pub(crate) fn new(fields: Fields, child_values: Vec>) -> Result { if fields.len() != child_values.len() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "StructConstant: fields length ({}) != child_values length ({})", - fields.len(), - child_values.len() - ), + return Err(invalid_data!( + "StructConstant: fields length ({}) != child_values length ({})", + fields.len(), + child_values.len() )); } Ok(Self { @@ -863,10 +861,7 @@ impl RecordBatchTransformer { // Iceberg-Java's Parquet readers (BaseParquetReaders / SparkParquetReaders), // which raise "Missing required field: ". if iceberg_field.initial_default.is_none() && iceberg_field.required { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Missing required field: {}", iceberg_field.name), - )); + return Err(invalid_data!("Missing required field: {}", iceberg_field.name)); } let default_value = iceberg_field.initial_default.as_ref().and_then(|lit| { @@ -895,12 +890,9 @@ impl RecordBatchTransformer { for (source_field_idx, source_field) in source_schema.fields.iter().enumerate() { // Check if field has a field ID in metadata if let Some(field_id_str) = source_field.metadata().get(PARQUET_FIELD_ID_META_KEY) { - let this_field_id = field_id_str.parse().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("field id not parseable as an i32: {e}"), - ) - })?; + let this_field_id = field_id_str + .parse() + .map_err(|e| invalid_data!("field id not parseable as an i32: {e}"))?; field_id_to_source_schema .insert(this_field_id, (source_field.clone(), source_field_idx)); @@ -998,9 +990,9 @@ impl RecordBatchTransformer { target_type: &DataType, ) -> Result { if source.data_type() != &DataType::Int64 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("coalesce source must be Int64, got {}", source.data_type()), + return Err(invalid_data!( + "coalesce source must be Int64, got {}", + source.data_type() )); } let PrimitiveLiteral::Long(seq) = fallback else { diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 4ef7b29436..e88c24d1a7 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -34,7 +34,7 @@ use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::file::statistics::Statistics; use uuid::Uuid; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::decimal_utils::i128_from_be_bytes; use crate::spec::{ Datum, FIRST_FIELD_ID, ListType, MapType, NestedField, NestedFieldRef, PrimitiveLiteral, @@ -206,10 +206,7 @@ fn visit_type(r#type: &DataType, visitor: &mut V) -> Resu DataType::Map(field, _) => match field.data_type() { DataType::Struct(fields) => { if fields.len() != 2 { - return Err(Error::new( - ErrorKind::DataInvalid, - "Map field must have exactly 2 fields", - )); + return Err(invalid_data!("Map field must have exactly 2 fields")); } let key_field = &fields[0]; @@ -231,17 +228,11 @@ fn visit_type(r#type: &DataType, visitor: &mut V) -> Resu visitor.map(r#type, key_result, value_result) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "Map field must have struct type", - )), + _ => Err(invalid_data!("Map field must have struct type")), }, DataType::Struct(fields) => visit_struct(fields, visitor), DataType::Dictionary(_key_type, value_type) => visit_type(value_type, visitor), - other => Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot visit Arrow data type: {other}"), - )), + other => Err(invalid_data!("Cannot visit Arrow data type: {other}")), } } @@ -329,18 +320,12 @@ const ARROW_FIELD_DOC_KEY: &str = "doc"; pub(super) fn get_field_id_from_metadata(field: &FieldRef) -> Result { if let Some(value) = field.metadata().get(PARQUET_FIELD_ID_META_KEY) { return value.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to parse field id".to_string(), - ) - .with_context("value", value) - .with_source(e) + invalid_data!("Failed to parse field id") + .with_context("value", value) + .with_source(e) }); } - Err(Error::new( - ErrorKind::DataInvalid, - "Field id not found in metadata", - )) + Err(invalid_data!("Field id not found in metadata")) } fn get_field_doc(field: &FieldRef) -> Option { @@ -439,10 +424,7 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::LargeList(element_field) => element_field, DataType::FixedSizeList(element_field, _) => element_field, _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - "List type must have list data type", - )); + return Err(invalid_data!("List type must have list data type")); } }; @@ -462,10 +444,7 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::Map(field, _) => match field.data_type() { DataType::Struct(fields) => { if fields.len() != 2 { - return Err(Error::new( - ErrorKind::DataInvalid, - "Map field must have exactly 2 fields", - )); + return Err(invalid_data!("Map field must have exactly 2 fields")); } let key_field = &fields[0]; @@ -496,15 +475,9 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { value_field, })) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "Map field must have struct type", - )), + _ => Err(invalid_data!("Map field must have struct type")), }, - _ => Err(Error::new( - ErrorKind::DataInvalid, - "Map type must have map data type", - )), + _ => Err(invalid_data!("Map type must have map data type")), } } @@ -520,20 +493,14 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::Int64 => Ok(Type::Primitive(PrimitiveType::Long)), DataType::UInt64 => { // Block uint64 - no safe casting option - Err(Error::new( - ErrorKind::DataInvalid, - "UInt64 is not supported. Use Int64 for values ≤ 9,223,372,036,854,775,807 or Decimal(20,0) for full uint64 range.", + Err(invalid_data!( + "UInt64 is not supported. Use Int64 for values ≤ 9,223,372,036,854,775,807 or Decimal(20,0) for full uint64 range." )) } DataType::Float32 => Ok(Type::Primitive(PrimitiveType::Float)), DataType::Float64 => Ok(Type::Primitive(PrimitiveType::Double)), - DataType::Decimal128(p, s) => Type::decimal(*p as u32, *s as u32).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to create decimal type".to_string(), - ) - .with_source(e) - }), + DataType::Decimal128(p, s) => Type::decimal(*p as u32, *s as u32) + .map_err(|e| invalid_data!("Failed to create decimal type").with_source(e)), DataType::Date32 => Ok(Type::Primitive(PrimitiveType::Date)), DataType::Time64(unit) if unit == &TimeUnit::Microsecond => { Ok(Type::Primitive(PrimitiveType::Time)) @@ -565,10 +532,7 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::Utf8View | DataType::Utf8 | DataType::LargeUtf8 => { Ok(Type::Primitive(PrimitiveType::String)) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Unsupported Arrow data type: {p}"), - )), + _ => Err(invalid_data!("Unsupported Arrow data type: {p}")), } } @@ -576,9 +540,8 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { // The extension may only sit on struct storage (mirrors // `VariantExtensionType::supports_data_type`). if !matches!(field.data_type(), DataType::Struct(_)) { - return Err(Error::new( - ErrorKind::DataInvalid, - "arrow.parquet.variant extension requires Struct storage", + return Err(invalid_data!( + "arrow.parquet.variant extension requires Struct storage" )); } // Fold the whole struct into a single logical variant without descending: @@ -708,28 +671,19 @@ impl SchemaVisitor for ToArrowSchemaConverter { PrimitiveType::Decimal { precision, scale } => { let (precision, scale) = { let precision: u8 = precision.to_owned().try_into().map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "incompatible precision for decimal type convert", - ) - .with_source(err) + invalid_data!("incompatible precision for decimal type convert") + .with_source(err) })?; let scale = scale.to_owned().try_into().map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "incompatible scale for decimal type convert", - ) - .with_source(err) + invalid_data!("incompatible scale for decimal type convert") + .with_source(err) })?; (precision, scale) }; validate_decimal_precision_and_scale::(precision, scale).map_err( |err| { - Error::new( - ErrorKind::DataInvalid, - "incompatible precision and scale for decimal type convert", - ) - .with_source(err) + invalid_data!("incompatible precision and scale for decimal type convert") + .with_source(err) }, )?; Ok(ArrowSchemaOrFieldOrType::Type(DataType::Decimal128( @@ -854,7 +808,7 @@ pub(crate) fn get_arrow_datum(datum: &Datum) -> Result { let array = FixedSizeBinaryArray::try_from_iter(std::iter::once(value.as_slice())) - .map_err(|e| Error::new(ErrorKind::DataInvalid, e.to_string()))?; + .map_err(|e| invalid_data!("FixedSizeBinary conversion failed").with_source(e))?; Ok(Arc::new(Scalar::new(array))) } @@ -941,12 +895,10 @@ pub(crate) fn get_parquet_stat_min_as_datum( }; Some(Datum::new( primitive_type.clone(), - PrimitiveLiteral::Int128(i128_from_be_bytes(bytes).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't convert bytes to i128: {bytes:?}"), - ) - })?), + PrimitiveLiteral::Int128( + i128_from_be_bytes(bytes) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), )) } ( @@ -1087,12 +1039,10 @@ pub(crate) fn get_parquet_stat_max_as_datum( }; Some(Datum::new( primitive_type.clone(), - PrimitiveLiteral::Int128(i128_from_be_bytes(bytes).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't convert bytes to i128: {bytes:?}"), - ) - })?), + PrimitiveLiteral::Int128( + i128_from_be_bytes(bytes) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), )) } ( diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index ad2a7f6476..080d1b99a6 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -28,6 +28,7 @@ use arrow_schema::{DataType, FieldRef, TimeUnit}; use uuid::Uuid; use super::get_field_id_from_metadata; +use crate::error::invalid_data; use crate::spec::{ ListType, Literal, Map, MapType, NestedField, PartnerAccessor, PrimitiveLiteral, PrimitiveType, SchemaWithPartnerVisitor, Struct, StructType, Type, VariantType, visit_struct_with_partner, @@ -57,12 +58,9 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { ) -> Result>> { // Make there is no null value if the field is required if field.required && value.iter().any(Option::is_none) { - return Err(Error::new( - ErrorKind::DataInvalid, - "The field is required but has null value", - ) - .with_context("field_id", field.id.to_string()) - .with_context("field_name", &field.name)); + return Err(invalid_data!("The field is required but has null value") + .with_context("field_id", field.id.to_string()) + .with_context("field_name", &field.name)); } Ok(value) } @@ -75,12 +73,11 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { ) -> Result>> { let row_len = results.first().map(|column| column.len()).unwrap_or(0); if let Some(col) = results.iter().find(|col| col.len() != row_len) { - return Err(Error::new( - ErrorKind::DataInvalid, - "The struct columns have different row length", - ) - .with_context("first col length", row_len.to_string()) - .with_context("actual col length", col.len().to_string())); + return Err( + invalid_data!("The struct columns have different row length") + .with_context("first col length", row_len.to_string()) + .with_context("actual col length", col.len().to_string()), + ); } let mut struct_literals = Vec::with_capacity(row_len); @@ -111,19 +108,14 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { elements: Vec>, ) -> Result>> { if list.element_field.required && elements.iter().any(Option::is_none) { - return Err(Error::new( - ErrorKind::DataInvalid, - "The list should not have null value", - )); + return Err(invalid_data!("The list should not have null value")); } match array.data_type() { DataType::List(_) => { let offset = array .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a list array") - })? + .ok_or_else(|| invalid_data!("The partner is not a list array"))? .offsets(); // combine the result according to the offset let mut result = Vec::with_capacity(offset.len() - 1); @@ -138,12 +130,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { let offset = array .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a large list array", - ) - })? + .ok_or_else(|| invalid_data!("The partner is not a large list array"))? .offsets(); // combine the result according to the offset let mut result = Vec::with_capacity(offset.len() - 1); @@ -163,10 +150,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { } Ok(result) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a list type", - )), + _ => Err(invalid_data!("The partner is not a list type")), } } @@ -179,16 +163,15 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { ) -> Result>> { // Make sure key_value and value have the same row length if key_values.len() != values.len() { - return Err(Error::new( - ErrorKind::DataInvalid, - "The key value and value of map should have the same row length", + return Err(invalid_data!( + "The key value and value of map should have the same row length" )); } let offsets = partner .as_any() .downcast_ref::() - .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "The partner is not a map array"))? + .ok_or_else(|| invalid_data!("The partner is not a map array"))? .offsets(); // combine the result according to the offset let mut result = Vec::with_capacity(offsets.len() - 1); @@ -211,65 +194,47 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a boolean array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a boolean array"))?; Ok(array.iter().map(|v| v.map(Literal::bool)).collect()) } PrimitiveType::Int => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a int32 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a int32 array"))?; Ok(array.iter().map(|v| v.map(Literal::int)).collect()) } PrimitiveType::Long => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a int64 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a int64 array"))?; Ok(array.iter().map(|v| v.map(Literal::long)).collect()) } PrimitiveType::Float => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a float32 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a float32 array"))?; Ok(array.iter().map(|v| v.map(Literal::float)).collect()) } PrimitiveType::Double => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a float64 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a float64 array"))?; Ok(array.iter().map(|v| v.map(Literal::double)).collect()) } PrimitiveType::Decimal { precision, scale } => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a decimal128 array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a decimal128 array"))?; if let DataType::Decimal128(arrow_precision, arrow_scale) = array.data_type() && (*arrow_precision as u32 != *precision || *arrow_scale as u32 != *scale) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "The precision or scale ({arrow_precision},{arrow_scale}) of arrow decimal128 array is not compatible with iceberg decimal type ({precision},{scale})" - ), + return Err(invalid_data!( + "The precision or scale ({arrow_precision},{arrow_scale}) of arrow decimal128 array is not compatible with iceberg decimal type ({precision},{scale})" )); } Ok(array.iter().map(|v| v.map(Literal::decimal)).collect()) @@ -278,54 +243,35 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a date32 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a date32 array"))?; Ok(array.iter().map(|v| v.map(Literal::date)).collect()) } PrimitiveType::Time => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a time64 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a time64 array"))?; Ok(array.iter().map(|v| v.map(Literal::time)).collect()) } PrimitiveType::Timestamp => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a timestamp array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a timestamp array"))?; Ok(array.iter().map(|v| v.map(Literal::timestamp)).collect()) } PrimitiveType::Timestamptz => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a timestamptz array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a timestamptz array"))?; Ok(array.iter().map(|v| v.map(Literal::timestamptz)).collect()) } PrimitiveType::TimestampNs => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a timestamp_ns array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a timestamp_ns array"))?; Ok(array .iter() .map(|v| v.map(Literal::timestamp_nano)) @@ -335,12 +281,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a timestamptz_ns array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a timestamptz_ns array"))?; Ok(array .iter() .map(|v| v.map(Literal::timestamptz_nano)) @@ -352,54 +293,37 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { } else if let Some(array) = partner.as_any().downcast_ref::() { Ok(array.iter().map(|v| v.map(Literal::string)).collect()) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a string array", - )) + Err(invalid_data!("The partner is not a string array")) } } PrimitiveType::Uuid => { if let Some(array) = partner.as_any().downcast_ref::() { if array.value_length() != 16 { - return Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a uuid array", - )); + return Err(invalid_data!("The partner is not a uuid array")); } Ok(array .iter() .map(|v| { v.map(|v| { Ok(Literal::uuid(Uuid::from_bytes(v.try_into().map_err( - |_| { - Error::new( - ErrorKind::DataInvalid, - "Failed to convert binary to uuid", - ) - }, + |_| invalid_data!("Failed to convert binary to uuid"), )?))) }) .transpose() }) .collect::>>()?) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a uuid array", - )) + Err(invalid_data!("The partner is not a uuid array")) } } PrimitiveType::Fixed(len) => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a fixed array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a fixed array"))?; if array.value_length() != *len as i32 { - return Err(Error::new( - ErrorKind::DataInvalid, - "The length of fixed size binary array is not compatible with iceberg fixed type", + return Err(invalid_data!( + "The length of fixed size binary array is not compatible with iceberg fixed type" )); } Ok(array @@ -419,10 +343,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { .map(|v| v.map(|v| Literal::binary(v.to_vec()))) .collect()) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a binary array", - )) + Err(invalid_data!("The partner is not a binary array")) } } } @@ -494,10 +415,7 @@ impl Default for ArrowArrayAccessor { impl PartnerAccessor for ArrowArrayAccessor { fn struct_partner<'a>(&self, schema_partner: &'a ArrayRef) -> Result<&'a ArrayRef> { if !matches!(schema_partner.data_type(), DataType::Struct(_)) { - return Err(Error::new( - ErrorKind::DataInvalid, - "The schema partner is not a struct type", - )); + return Err(invalid_data!("The schema partner is not a struct type")); } Ok(schema_partner) @@ -512,11 +430,8 @@ impl PartnerAccessor for ArrowArrayAccessor { .as_any() .downcast_ref::() .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "The struct partner is not a struct array, partner: {struct_partner:?}" - ), + invalid_data!( + "The struct partner is not a struct array, partner: {struct_partner:?}" ) })?; @@ -524,12 +439,7 @@ impl PartnerAccessor for ArrowArrayAccessor { .fields() .iter() .position(|arrow_field| self.match_mode.match_field(arrow_field, field)) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Field id {} not found in struct array", field.id), - ) - })?; + .ok_or_else(|| invalid_data!("Field id {} not found in struct array", field.id))?; Ok(struct_array.column(field_pos)) } @@ -540,24 +450,14 @@ impl PartnerAccessor for ArrowArrayAccessor { let list_array = list_partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The list partner is not a list array", - ) - })?; + .ok_or_else(|| invalid_data!("The list partner is not a list array"))?; Ok(list_array.values()) } DataType::LargeList(_) => { let list_array = list_partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The list partner is not a large list array", - ) - })?; + .ok_or_else(|| invalid_data!("The list partner is not a large list array"))?; Ok(list_array.values()) } DataType::FixedSizeList(_, _) => { @@ -565,17 +465,11 @@ impl PartnerAccessor for ArrowArrayAccessor { .as_any() .downcast_ref::() .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The list partner is not a fixed size list array", - ) + invalid_data!("The list partner is not a fixed size list array") })?; Ok(list_array.values()) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "The list partner is not a list type", - )), + _ => Err(invalid_data!("The list partner is not a list type")), } } @@ -583,9 +477,7 @@ impl PartnerAccessor for ArrowArrayAccessor { let map_array = map_partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The map partner is not a map array") - })?; + .ok_or_else(|| invalid_data!("The map partner is not a map array"))?; Ok(map_array.keys()) } @@ -593,9 +485,7 @@ impl PartnerAccessor for ArrowArrayAccessor { let map_array = map_partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The map partner is not a map array") - })?; + .ok_or_else(|| invalid_data!("The map partner is not a map array"))?; Ok(map_array.values()) } } @@ -708,12 +598,9 @@ pub(crate) fn create_primitive_array_single_element( let array = Decimal128Array::from(vec![{ *v }]) .with_precision_and_scale(*precision, *scale) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" - ), - ) + ) })?; Ok(Arc::new(array)) } @@ -721,12 +608,9 @@ pub(crate) fn create_primitive_array_single_element( let array = Decimal128Array::from(vec![*v as i128]) .with_precision_and_scale(*precision, *scale) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" - ), - ) + ) })?; Ok(Arc::new(array)) } @@ -734,12 +618,9 @@ pub(crate) fn create_primitive_array_single_element( let array = Decimal128Array::from(vec![Option::::None]) .with_precision_and_scale(*precision, *scale) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" - ), - ) + ) })?; Ok(Arc::new(array)) } @@ -885,22 +766,16 @@ pub(crate) fn create_primitive_array_repeated( // wrong-width literal would otherwise produce a FixedSizeBinary of the // wrong width instead of erroring. if value.len() != *len as usize { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "FixedSizeBinary literal length {} does not match declared width {len}", - value.len() - ), + return Err(invalid_data!( + "FixedSizeBinary literal length {} does not match declared width {len}", + value.len() )); } Arc::new( FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n(value.as_slice(), num_rows)) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to create FixedSizeBinary({len}) array: {e}"), - ) + invalid_data!("Failed to create FixedSizeBinary({len}) array: {e}") })?, ) } @@ -912,12 +787,9 @@ pub(crate) fn create_primitive_array_repeated( Decimal128Array::from(vec![*value; num_rows]) .with_precision_and_scale(*precision, *scale) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" - ), - ) + ) })?, ) } @@ -926,12 +798,9 @@ pub(crate) fn create_primitive_array_repeated( Decimal128Array::from(vec![*value as i128; num_rows]) .with_precision_and_scale(*precision, *scale) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" - ), - ) + ) })?, ) } @@ -943,12 +812,9 @@ pub(crate) fn create_primitive_array_repeated( Decimal128Array::from(vals) .with_precision_and_scale(*precision, *scale) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" - ), - ) + ) })?, ) } diff --git a/crates/iceberg/src/avro/schema.rs b/crates/iceberg/src/avro/schema.rs index a67e4ed71b..908dff5c9f 100644 --- a/crates/iceberg/src/avro/schema.rs +++ b/crates/iceberg/src/avro/schema.rs @@ -26,6 +26,7 @@ use apache_avro::schema::{ use itertools::{Either, Itertools}; use serde_json::{Number, Value}; +use crate::error::invalid_data; use crate::spec::{ ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, Schema, SchemaVisitor, StructType, Type, VariantType, visit_schema, @@ -401,9 +402,8 @@ pub(crate) fn visit(schema: &AvroSchema, visitor: &mut V) let value = visit(&record_schema.fields[1].schema, visitor)?; return visitor.map_array(record_schema, key, value); } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Can't convert avro map schema, item is not a record.", + return Err(invalid_data!( + "Can't convert avro map schema, item is not a record." )); } } else { @@ -437,25 +437,16 @@ impl AvroSchemaToSchema { ) -> Result { attributes .get(name) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro array schema, missing element id.", - ) - })? + .ok_or_else(|| invalid_data!("Can't convert avro array schema, missing element id."))? .as_i64() .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro array schema, element id is not a valid i64 number.", + invalid_data!( + "Can't convert avro array schema, element id is not a valid i64 number." ) })? .try_into() .map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro array schema, element id is not a valid i32.", - ) + invalid_data!("Can't convert avro array schema, element id is not a valid i32.") }) } } @@ -580,12 +571,8 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { key: Option, value: Option, ) -> Result { - let key = key.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro map schema, missing key schema.", - ) - })?; + let key = + key.ok_or_else(|| invalid_data!("Can't convert avro map schema, missing key schema."))?; let value = unwrap_or_unknown(value); let key_id = Self::get_element_id_from_attributes( &array.fields[0].custom_attributes, @@ -628,9 +615,8 @@ pub(crate) fn avro_schema_to_schema(avro_schema: &AvroSchema) -> Result )) } } else { - Err(Error::new( - ErrorKind::DataInvalid, - "Can't convert non record avro schema to iceberg schema: {avro_schema}", + Err(invalid_data!( + "Can't convert non record avro schema to iceberg schema: {avro_schema}" )) } } @@ -1242,6 +1228,23 @@ mod tests { assert!(avro_schema_to_schema(&avro_schema).is_err()); } + #[test] + fn test_non_record_schema_error_interpolates_schema() { + let err = avro_schema_to_schema(&AvroSchema::String).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + !err.message().contains("{avro_schema}"), + "message should interpolate the schema, got: {}", + err.message() + ); + assert!( + err.message().ends_with(&AvroSchema::String.to_string()), + "message should end with the rendered schema, got: {}", + err.message() + ); + } + #[test] fn test_decimal_type() { let avro_schema = { diff --git a/crates/iceberg/src/catalog/memory/catalog.rs b/crates/iceberg/src/catalog/memory/catalog.rs index aae0f7d5ac..dc5079a1ae 100644 --- a/crates/iceberg/src/catalog/memory/catalog.rs +++ b/crates/iceberg/src/catalog/memory/catalog.rs @@ -28,13 +28,14 @@ use itertools::Itertools; use super::namespace_state::NamespaceState; use crate::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use crate::error::invalid_data; use crate::io::{FileIO, FileIOBuilder, MemoryStorageFactory, StorageFactory}; use crate::runtime::Runtime; use crate::spec::{TableMetadata, TableMetadataBuilder}; use crate::table::Table; use crate::{ - Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, - TableCommit, TableCreation, TableIdent, + Catalog, CatalogBuilder, MetadataLocation, Namespace, NamespaceIdent, Result, TableCommit, + TableCreation, TableIdent, }; /// Memory catalog warehouse location @@ -79,10 +80,7 @@ impl CatalogBuilder for MemoryCatalogBuilder { async move { let catalog_properties = MemoryCatalogProperties::from_properties(&props)?; if catalog_properties.warehouse.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Catalog warehouse is required", - )); + return Err(invalid_data!("Catalog warehouse is required")); } let runtime = self.runtime.unwrap_or_else(Runtime::current); @@ -438,6 +436,7 @@ pub(crate) mod tests { use tempfile::TempDir; use super::*; + use crate::ErrorKind; use crate::encryption::kms::MemoryKmsClientFactory; use crate::io::{FileIO, LocalFsStorageFactory}; use crate::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type}; diff --git a/crates/iceberg/src/catalog/memory/namespace_state.rs b/crates/iceberg/src/catalog/memory/namespace_state.rs index d7dd6c4b2e..d8361dd246 100644 --- a/crates/iceberg/src/catalog/memory/namespace_state.rs +++ b/crates/iceberg/src/catalog/memory/namespace_state.rs @@ -19,6 +19,7 @@ use std::collections::{HashMap, hash_map}; use itertools::Itertools; +use crate::error::invalid_data; use crate::table::Table; use crate::{Error, ErrorKind, NamespaceIdent, Result, TableIdent}; @@ -111,10 +112,7 @@ impl NamespaceState { namespace_ident: &NamespaceIdent, ) -> Result<(&mut NamespaceState, String)> { match namespace_ident.split_last() { - None => Err(Error::new( - ErrorKind::DataInvalid, - "Namespace identifier can't be empty!", - )), + None => Err(invalid_data!("Namespace identifier can't be empty!")), Some((child_namespace_name, parent_name_parts)) => { let parent_namespace_state = if parent_name_parts.is_empty() { Ok(self) diff --git a/crates/iceberg/src/catalog/mod.rs b/crates/iceberg/src/catalog/mod.rs index d2ecc7d51e..cf7a94d20f 100644 --- a/crates/iceberg/src/catalog/mod.rs +++ b/crates/iceberg/src/catalog/mod.rs @@ -42,6 +42,7 @@ use typed_builder::TypedBuilder; use uuid::Uuid; use crate::encryption::kms::KmsClientFactory; +use crate::error::invalid_data; use crate::io::StorageFactory; use crate::runtime::Runtime; use crate::spec::{ @@ -209,10 +210,7 @@ impl NamespaceIdent { /// Create a multi-level namespace identifier from vector. pub fn from_vec(names: Vec) -> Result { if names.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Namespace identifier can't be empty!", - )); + return Err(invalid_data!("Namespace identifier can't be empty!")); } Ok(Self(names)) } @@ -322,9 +320,9 @@ impl TableIdent { /// Try to create table identifier from an iterator of string. pub fn from_strs(iter: impl IntoIterator) -> Result { let mut vec: Vec = iter.into_iter().map(|s| s.to_string()).collect(); - let table_name = vec.pop().ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "Table identifier can't be empty!") - })?; + let table_name = vec + .pop() + .ok_or_else(|| invalid_data!("Table identifier can't be empty!"))?; let namespace_ident = NamespaceIdent::from_vec(vec)?; Ok(Self { diff --git a/crates/iceberg/src/delete_file_index.rs b/crates/iceberg/src/delete_file_index.rs index 2e749cab1f..692ab9315a 100644 --- a/crates/iceberg/src/delete_file_index.rs +++ b/crates/iceberg/src/delete_file_index.rs @@ -23,11 +23,12 @@ use futures::StreamExt; use futures::channel::mpsc::{Sender, channel}; use tokio::sync::Notify; +use crate::error::invalid_data; use crate::metadata_columns::RESERVED_FIELD_ID_DELETE_FILE_PATH; use crate::runtime::Runtime; use crate::scan::{DeleteFileContext, FileScanTaskDeleteFile}; use crate::spec::{DataContentType, DataFile, DataFileFormat, PrimitiveLiteral, Struct}; -use crate::{Error, ErrorKind, Result}; +use crate::{Error, Result}; /// Index of delete files #[derive(Debug, Clone)] @@ -219,24 +220,18 @@ impl PopulatedDeleteFileIndex { // malformed manifest entry, not an ordinary position delete to fall back // on. let Some(path) = data_file.referenced_data_file() else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} is missing referenced_data_file", - arc_ctx.manifest_entry.file_path() - ), + return Err(invalid_data!( + "deletion vector {} is missing referenced_data_file", + arc_ctx.manifest_entry.file_path() )); }; if data_file.content_offset().is_none() || data_file.content_size_in_bytes().is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} is missing content_offset or content_size_in_bytes", - arc_ctx.manifest_entry.file_path() - ), + return Err(invalid_data!( + "deletion vector {} is missing content_offset or content_size_in_bytes", + arc_ctx.manifest_entry.file_path() )); } @@ -244,13 +239,10 @@ impl PopulatedDeleteFileIndex { dvs_by_referenced_data_file.insert(path.clone(), arc_ctx) { let inserted = &dvs_by_referenced_data_file[&path]; - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "found multiple deletion vectors for data file {path}: {} and {}", - existing.manifest_entry.file_path(), - inserted.manifest_entry.file_path() - ), + return Err(invalid_data!( + "found multiple deletion vectors for data file {path}: {} and {}", + existing.manifest_entry.file_path(), + inserted.manifest_entry.file_path() )); } continue; @@ -339,29 +331,23 @@ impl PopulatedDeleteFileIndex { if data_file.partition() != dv_data_file.partition() || data_file.partition_spec_id != dv.partition_spec_id { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} references data file {} but its partition (spec {}, {:?}) does not match the data file's partition (spec {}, {:?})", - dv.manifest_entry.file_path(), - data_file.file_path(), - dv.partition_spec_id, - dv_data_file.partition(), - data_file.partition_spec_id, - data_file.partition() - ), + return Err(invalid_data!( + "deletion vector {} references data file {} but its partition (spec {}, {:?}) does not match the data file's partition (spec {}, {:?})", + dv.manifest_entry.file_path(), + data_file.file_path(), + dv.partition_spec_id, + dv_data_file.partition(), + data_file.partition_spec_id, + data_file.partition() )); } if let Some(seq_num) = seq_num { let dv_seq = dv.manifest_entry.sequence_number(); if dv_seq < Some(seq_num) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion vector {} has data sequence number {dv_seq:?}, which must be >= the data file's sequence number {seq_num}", - dv.manifest_entry.file_path() - ), + return Err(invalid_data!( + "deletion vector {} has data sequence number {dv_seq:?}, which must be >= the data file's sequence number {seq_num}", + dv.manifest_entry.file_path() )); } } @@ -406,6 +392,7 @@ mod tests { use uuid::Uuid; use super::*; + use crate::ErrorKind; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Datum, Literal, ManifestEntry, ManifestStatus, Struct, diff --git a/crates/iceberg/src/delete_vector.rs b/crates/iceberg/src/delete_vector.rs index 4ab13d52b1..9b2d48d91a 100644 --- a/crates/iceberg/src/delete_vector.rs +++ b/crates/iceberg/src/delete_vector.rs @@ -22,6 +22,7 @@ use roaring::bitmap::Iter; use roaring::treemap::BitmapIter; use roaring::{RoaringBitmap, RoaringTreemap}; +use crate::error::invalid_data; use crate::{Error, ErrorKind, Result}; /// Magic bytes prefixing a serialized `deletion-vector-v1` bitmap, per the Iceberg Puffin spec. @@ -98,12 +99,9 @@ impl DeleteVector { /// comparison, or the roaring payload fails to decode. pub fn deserialize(blob: &[u8]) -> Result { if blob.len() < DV_MIN_BLOB_BYTES { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion-vector-v1 blob is {} bytes, shorter than the {DV_MIN_BLOB_BYTES}-byte minimum", - blob.len() - ), + return Err(invalid_data!( + "deletion-vector-v1 blob is {} bytes, shorter than the {DV_MIN_BLOB_BYTES}-byte minimum", + blob.len() )); } @@ -133,12 +131,9 @@ fn verify_length_prefix(mut prefix: &[u8], body: &[u8]) -> Result<()> { .with_source(e) })? as usize; if declared_len != body.len() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion-vector-v1 length prefix is {declared_len}, expected {}", - body.len() - ), + return Err(invalid_data!( + "deletion-vector-v1 length prefix is {declared_len}, expected {}", + body.len() )); } Ok(()) @@ -154,11 +149,8 @@ fn verify_crc(body: &[u8], mut crc_bytes: &[u8]) -> Result<()> { })?; let computed_crc = crc32fast::hash(body); if computed_crc != stored_crc { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion-vector-v1 CRC mismatch: computed {computed_crc:#010x}, stored {stored_crc:#010x}" - ), + return Err(invalid_data!( + "deletion-vector-v1 CRC mismatch: computed {computed_crc:#010x}, stored {stored_crc:#010x}" )); } Ok(()) @@ -166,9 +158,8 @@ fn verify_crc(body: &[u8], mut crc_bytes: &[u8]) -> Result<()> { fn verify_magic(magic: &[u8]) -> Result<()> { if magic != DV_MAGIC { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("deletion-vector-v1 magic mismatch: {magic:02x?}, expected {DV_MAGIC:02x?}"), + return Err(invalid_data!( + "deletion-vector-v1 magic mismatch: {magic:02x?}, expected {DV_MAGIC:02x?}" )); } Ok(()) @@ -182,21 +173,14 @@ fn verify_magic(magic: &[u8]) -> Result<()> { // that doesn't match what was actually written. fn decode_roaring_directory(mut reader: &[u8]) -> Result { let bitmap_count = reader.try_get_u64_le().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "failed to decode deletion-vector-v1 roaring payload", - ) - .with_source(e) + invalid_data!("failed to decode deletion-vector-v1 roaring payload").with_source(e) })?; // The roaring portable format restricts the bitmap count to [0, 2^32 - 1] (it is stored as a // u64 with the upper 32 bits reserved as zero padding). if bitmap_count > u32::MAX as u64 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion-vector-v1 roaring bitmap count {bitmap_count} exceeds the {}-key maximum", - u32::MAX - ), + return Err(invalid_data!( + "deletion-vector-v1 roaring bitmap count {bitmap_count} exceeds the {}-key maximum", + u32::MAX )); } @@ -204,30 +188,19 @@ fn decode_roaring_directory(mut reader: &[u8]) -> Result { let mut last_key: Option = None; for _ in 0..bitmap_count { let key = reader.try_get_u32_le().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "failed to decode deletion-vector-v1 roaring payload", - ) - .with_source(e) + invalid_data!("failed to decode deletion-vector-v1 roaring payload").with_source(e) })?; if let Some(last) = last_key && key <= last { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "deletion-vector-v1 roaring keys must be ordered by unsigned comparison, got key {key} after {last}" - ), + return Err(invalid_data!( + "deletion-vector-v1 roaring keys must be ordered by unsigned comparison, got key {key} after {last}" )); } last_key = Some(key); let bitmap = RoaringBitmap::deserialize_from(&mut reader).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "failed to decode deletion-vector-v1 roaring payload", - ) - .with_source(e) + invalid_data!("failed to decode deletion-vector-v1 roaring payload").with_source(e) })?; bitmaps.push((key, bitmap)); } diff --git a/crates/iceberg/src/encryption/crypto.rs b/crates/iceberg/src/encryption/crypto.rs index babdb33bce..27ae9a336f 100644 --- a/crates/iceberg/src/encryption/crypto.rs +++ b/crates/iceberg/src/encryption/crypto.rs @@ -28,6 +28,7 @@ use aes_gcm::{Aes128Gcm, Aes256Gcm, AesGcm, Nonce}; /// from the underlying primitives, same as `Aes128Gcm` and `Aes256Gcm`. type Aes192Gcm = AesGcm; +use crate::error::invalid_data; use crate::sensitive::SensitiveBytes; use crate::{Error, ErrorKind, Result}; @@ -65,9 +66,8 @@ impl AesKeySize { 16 => Ok(Self::Bits128), 24 => Ok(Self::Bits192), 32 => Ok(Self::Bits256), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid data key length: {len} (must be 16, 24, or 32)"), + _ => Err(invalid_data!( + "Invalid data key length: {len} (must be 16, 24, or 32)" )), } } @@ -81,10 +81,7 @@ impl FromStr for AesKeySize { "128" | "AES_GCM_128" | "AES128_GCM" => Ok(Self::Bits128), "192" | "AES_GCM_192" | "AES192_GCM" => Ok(Self::Bits192), "256" | "AES_GCM_256" | "AES256_GCM" => Ok(Self::Bits256), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid AES key size: {s}"), - )), + _ => Err(invalid_data!("Invalid AES key size: {s}")), } } } @@ -195,13 +192,10 @@ impl AesGcmCipher { /// The decrypted plaintext. pub fn decrypt(&self, ciphertext: &[u8], aad: Option<&[u8]>) -> Result> { if ciphertext.len() < Self::NONCE_LEN + Self::TAG_LEN { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Ciphertext too short: expected at least {} bytes, got {}", - Self::NONCE_LEN + Self::TAG_LEN, - ciphertext.len() - ), + return Err(invalid_data!( + "Ciphertext too short: expected at least {} bytes, got {}", + Self::NONCE_LEN + Self::TAG_LEN, + ciphertext.len() )); } @@ -221,9 +215,8 @@ impl AesGcmCipher { fn encrypt_aes_gcm(key_bytes: &[u8], plaintext: &[u8], aad: Option<&[u8]>) -> Result> where C: Aead + AeadCore + KeyInit { - let cipher = C::new_from_slice(key_bytes).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Invalid AES key").with_source(anyhow::anyhow!(e)) - })?; + let cipher = C::new_from_slice(key_bytes) + .map_err(|e| invalid_data!("Invalid AES key").with_source(anyhow::anyhow!(e)))?; let nonce = C::generate_nonce(&mut OsRng); let ciphertext = if let Some(aad) = aad { @@ -248,9 +241,8 @@ where C: Aead + AeadCore + KeyInit { fn decrypt_aes_gcm(key_bytes: &[u8], ciphertext: &[u8], aad: Option<&[u8]>) -> Result> where C: Aead + AeadCore + KeyInit { - let cipher = C::new_from_slice(key_bytes).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Invalid AES key").with_source(anyhow::anyhow!(e)) - })?; + let cipher = C::new_from_slice(key_bytes) + .map_err(|e| invalid_data!("Invalid AES key").with_source(anyhow::anyhow!(e)))?; let nonce = Nonce::from_slice(&ciphertext[..AesGcmCipher::NONCE_LEN]); let encrypted_data = &ciphertext[AesGcmCipher::NONCE_LEN..]; diff --git a/crates/iceberg/src/encryption/key_metadata.rs b/crates/iceberg/src/encryption/key_metadata.rs index 765d600b50..4f8445a9a2 100644 --- a/crates/iceberg/src/encryption/key_metadata.rs +++ b/crates/iceberg/src/encryption/key_metadata.rs @@ -24,6 +24,7 @@ use aes_gcm::aead::OsRng; use aes_gcm::aead::rand_core::RngCore; use super::{AesKeySize, SecureKey}; +use crate::error::invalid_data; use crate::{Error, ErrorKind, Result}; /// Standard key metadata for Iceberg table encryption. @@ -196,10 +197,7 @@ mod _serde { pub(super) fn decode(bytes: &[u8]) -> Result { if bytes.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Empty key metadata buffer", - )); + return Err(invalid_data!("Empty key metadata buffer")); } let version = bytes[0]; @@ -211,17 +209,11 @@ mod _serde { } let mut reader = Cursor::new(&bytes[1..]); - let value = from_avro_datum(&AVRO_SCHEMA_V1, &mut reader, None).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Failed to decode key metadata").with_source(e) - })?; + let value = from_avro_datum(&AVRO_SCHEMA_V1, &mut reader, None) + .map_err(|e| invalid_data!("Failed to decode key metadata").with_source(e))?; - from_value(&value).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to decode key metadata fields", - ) - .with_source(e) - }) + from_value(&value) + .map_err(|e| invalid_data!("Failed to decode key metadata fields").with_source(e)) } } @@ -243,11 +235,7 @@ mod _serde { fn try_from(v1: StandardKeyMetadataV1) -> Result { let encryption_key = SecureKey::new(&v1.encryption_key).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Invalid encryption key in key metadata", - ) - .with_source(e) + invalid_data!("Invalid encryption key in key metadata").with_source(e) })?; Ok(Self { encryption_key, diff --git a/crates/iceberg/src/encryption/kms/memory.rs b/crates/iceberg/src/encryption/kms/memory.rs index 4c95b8421a..3950b11fbf 100644 --- a/crates/iceberg/src/encryption/kms/memory.rs +++ b/crates/iceberg/src/encryption/kms/memory.rs @@ -29,7 +29,7 @@ use async_trait::async_trait; use super::KeyManagementClient; use super::factory::KmsClientFactory; use crate::encryption::{AesGcmCipher, AesKeySize, SecureKey, SensitiveBytes}; -use crate::error::lock_error; +use crate::error::{invalid_data, lock_error}; use crate::{Error, ErrorKind, Result}; /// In-memory KMS for testing. Not suitable for production use. @@ -108,10 +108,7 @@ impl MemoryKeyManagementClient { let mut keys = self.master_keys.write().map_err(lock_error)?; if keys.contains_key(&key_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Master key already exists: {key_id}"), - )); + return Err(invalid_data!("Master key already exists: {key_id}")); } keys.insert(key_id, key); @@ -121,12 +118,9 @@ impl MemoryKeyManagementClient { fn get_master_key(&self, key_id: &str) -> Result { let keys = self.master_keys.read().map_err(lock_error)?; - keys.get(key_id).cloned().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Master key not found: {key_id}"), - ) - }) + keys.get(key_id) + .cloned() + .ok_or_else(|| invalid_data!("Master key not found: {key_id}")) } /// Number of master keys stored. diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index f54ef464d5..d6f4e4af6b 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -38,6 +38,7 @@ use super::crypto::{AesGcmCipher, AesKeySize, SecureKey}; use super::io::EncryptedOutputFile; use super::key_metadata::StandardKeyMetadata; use super::kms::KeyManagementClient; +use crate::error::invalid_data; use crate::io::OutputFile; use crate::sensitive::SensitiveBytes; use crate::spec::{EncryptedKey, FormatVersion, TableMetadataRef}; @@ -204,20 +205,12 @@ impl EncryptionManager { .expect("encryption_keys lock poisoned") .get(encryption_key_id) .cloned() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Encryption key '{encryption_key_id}' not found"), - ) - })?; + .ok_or_else(|| invalid_data!("Encryption key '{encryption_key_id}' not found"))?; let kek_key_id = encrypted_key.encrypted_by_id().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "EncryptedKey '{}' has no encrypted_by_id", - encrypted_key.key_id() - ), + invalid_data!( + "EncryptedKey '{}' has no encrypted_by_id", + encrypted_key.key_id() ) })?; @@ -330,12 +323,9 @@ impl EncryptionManager { return Ok(cached); } - let master_key_id = kek.encrypted_by_id().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("KEK '{}' has no encrypted_by_id", kek.key_id()), - ) - })?; + let master_key_id = kek + .encrypted_by_id() + .ok_or_else(|| invalid_data!("KEK '{}' has no encrypted_by_id", kek.key_id()))?; let plaintext = self .kms_client @@ -356,12 +346,7 @@ impl EncryptionManager { .expect("encryption_keys lock poisoned") .get(kek_key_id) .cloned() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("KEK not found in encryption keys: {kek_key_id}"), - ) - })?; + .ok_or_else(|| invalid_data!("KEK not found in encryption keys: {kek_key_id}"))?; // KEK timestamp as AAD prevents timestamp tampering. let aad = Self::kek_timestamp_aad(&kek)?; @@ -383,13 +368,10 @@ impl EncryptionManager { .get(KEK_CREATED_AT_PROPERTY) .map(|ts| ts.as_bytes()) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "KEK '{}' is missing required '{}' property", - kek.key_id(), - KEK_CREATED_AT_PROPERTY - ), + invalid_data!( + "KEK '{}' is missing required '{}' property", + kek.key_id(), + KEK_CREATED_AT_PROPERTY ) }) } diff --git a/crates/iceberg/src/encryption/stream.rs b/crates/iceberg/src/encryption/stream.rs index 7971a3df54..c14905f70e 100644 --- a/crates/iceberg/src/encryption/stream.rs +++ b/crates/iceberg/src/encryption/stream.rs @@ -49,6 +49,7 @@ use std::sync::Arc; use bytes::{Bytes, BytesMut}; use super::AesGcmCipher; +use crate::error::invalid_data; use crate::io::{FileRead, FileWrite}; use crate::{Error, ErrorKind, Result}; @@ -171,12 +172,9 @@ impl AesGcmFileRead { }; if num_blocks > u32::MAX as u64 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "AGS1 format supports at most {} blocks (~4 TiB per file), but file requires {num_blocks} blocks", - u32::MAX - ), + return Err(invalid_data!( + "AGS1 format supports at most {} blocks (~4 TiB per file), but file requires {num_blocks} blocks", + u32::MAX )); } @@ -207,11 +205,8 @@ impl AesGcmFileRead { /// `AesGcmInputStream.calculatePlaintextLength()`. pub fn calculate_plaintext_length(encrypted_file_length: u64) -> Result { if encrypted_file_length < GCM_STREAM_HEADER_LENGTH as u64 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Encrypted file too short: {encrypted_file_length} bytes (minimum {GCM_STREAM_HEADER_LENGTH})" - ), + return Err(invalid_data!( + "Encrypted file too short: {encrypted_file_length} bytes (minimum {GCM_STREAM_HEADER_LENGTH})" )); } @@ -229,13 +224,10 @@ impl AesGcmFileRead { 0 } else { if cipher_bytes_in_last_block < (NONCE_LENGTH + GCM_TAG_LENGTH) as u64 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Truncated encrypted file: last block is {} bytes (minimum {})", - cipher_bytes_in_last_block, - NONCE_LENGTH + GCM_TAG_LENGTH - ), + return Err(invalid_data!( + "Truncated encrypted file: last block is {} bytes (minimum {})", + cipher_bytes_in_last_block, + NONCE_LENGTH + GCM_TAG_LENGTH )); } cipher_bytes_in_last_block - NONCE_LENGTH as u64 - GCM_TAG_LENGTH as u64 @@ -293,22 +285,19 @@ impl FileRead for AesGcmFileRead { } if range.start > range.end { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid read range: start ({}) is greater than end ({})", - range.start, range.end - ), + return Err(invalid_data!( + "Invalid read range: start ({}) is greater than end ({})", + range.start, + range.end )); } if range.end > self.plain_stream_size { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Read range {}..{} exceeds plaintext size {}", - range.start, range.end, self.plain_stream_size - ), + return Err(invalid_data!( + "Read range {}..{} exceeds plaintext size {}", + range.start, + range.end, + self.plain_stream_size )); } @@ -442,9 +431,8 @@ impl AesGcmFileWrite { return Err(e); } self.block_index = self.block_index.checked_add(1).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "AGS1 block index overflow: file exceeds the maximum supported size (~4 TiB)", + invalid_data!( + "AGS1 block index overflow: file exceeds the maximum supported size (~4 TiB)" ) })?; Ok(()) @@ -462,9 +450,8 @@ impl AesGcmFileWrite { return Err(e); } self.block_index = self.block_index.checked_add(1).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "AGS1 block index overflow: file exceeds the maximum supported size (~4 TiB)", + invalid_data!( + "AGS1 block index overflow: file exceeds the maximum supported size (~4 TiB)" ) })?; self.buffer.drain(..PLAIN_BLOCK_SIZE as usize); @@ -600,14 +587,11 @@ mod tests { let start = range.start as usize; let end = range.end as usize; if end > self.0.len() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Range {}..{} out of bounds for {} bytes", - start, - end, - self.0.len() - ), + return Err(invalid_data!( + "Range {}..{} out of bounds for {} bytes", + start, + end, + self.0.len() )); } Ok(self.0.slice(start..end)) diff --git a/crates/iceberg/src/error.rs b/crates/iceberg/src/error.rs index 02c3eee8fc..2f5c92894d 100644 --- a/crates/iceberg/src/error.rs +++ b/crates/iceberg/src/error.rs @@ -469,6 +469,47 @@ macro_rules! ensure_data_valid { }; } +/// Helper macro to construct an [`ErrorKind::DataInvalid`] error. +/// +/// This is a shorthand for `Error::new(ErrorKind::DataInvalid, ...)`, the most +/// common error constructed in this crate. It returns the [`Error`] value (it +/// does *not* return from the enclosing function), so it composes with `?`, +/// `.map_err(...)`, `.ok_or_else(...)`, and explicit `return Err(...)`. +/// +/// The message may be a plain expression or a format string with arguments. +/// +/// Unlike the public [`ensure_data_valid!`], this macro is deliberately +/// crate-internal — adding `#[macro_export]` would commit it to the public API. +/// +/// # Examples +/// +/// The `use` path below is crate-internal and only resolves inside this crate. +/// +/// ```ignore +/// use crate::error::invalid_data; +/// +/// // As an expression +/// let err = invalid_data!("unexpected value: {value}"); +/// +/// // With `.ok_or_else` +/// let field = fields.get(id).ok_or_else(|| invalid_data!("missing field {id}"))?; +/// +/// // Attaching a source error +/// let n: i32 = s.parse().map_err(|e| invalid_data!("not an int: {s}").with_source(e))?; +/// ``` +macro_rules! invalid_data { + // Bare literals stay in `format!` so inline captures like `{id}` still interpolate. + ($fmt: literal $(, $($arg:tt)*)?) => { + $crate::error::Error::new($crate::error::ErrorKind::DataInvalid, format!($fmt $(, $($arg)*)?)) + }; + ($msg: expr $(,)?) => { + $crate::error::Error::new($crate::error::ErrorKind::DataInvalid, $msg) + }; +} + +// Crate-internal macro: re-exported so other modules can `use crate::error::invalid_data;`. +pub(crate) use invalid_data; + #[cfg(test)] mod tests { use anyhow::anyhow; @@ -532,6 +573,36 @@ Source: networking error ) } + #[test] + fn test_invalid_data_macro() { + // Plain literal. + let err = invalid_data!("something is wrong"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "something is wrong"); + + // Literal with an inline capture. + let value = 42; + let err = invalid_data!("unexpected value: {value}"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "unexpected value: 42"); + + // Literal with trailing arguments. + let err = invalid_data!("unexpected value: {}", value); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "unexpected value: 42"); + + // Non-literal message expression. + let computed = String::from("computed"); + let err = invalid_data!(computed); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "computed"); + + // Composes with builder methods like `.with_source`. + let err = invalid_data!("wrapping").with_source(anyhow!("root cause")); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.source.is_some()); + } + /// Backtrace contains build information, so we just assert the header of error content. #[test] fn test_error_debug_with_backtrace() { diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index 5a7330c8ca..f8ea896395 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -27,7 +27,7 @@ use fnv::FnvHashSet; use itertools::Itertools; use serde::{Deserialize, Serialize}; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::expr::visitors::bound_predicate_visitor::visit as visit_bound; use crate::expr::visitors::predicate_visitor::visit; use crate::expr::visitors::rewrite_not::RewriteNotVisitor; @@ -398,12 +398,9 @@ impl Bind for Predicate { } &PredicateOperator::IsNan | &PredicateOperator::NotNan => { if !bound_expr.term.field().field_type.is_floating_type() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Expecting floating point type, but found {}", - bound_expr.term.field().field_type - ), + return Err(invalid_data!( + "Expecting floating point type, but found {}", + bound_expr.term.field().field_type )); } } diff --git a/crates/iceberg/src/expr/term.rs b/crates/iceberg/src/expr/term.rs index ecdb88912b..4a5aedf743 100644 --- a/crates/iceberg/src/expr/term.rs +++ b/crates/iceberg/src/expr/term.rs @@ -22,12 +22,12 @@ use std::fmt::{Display, Formatter}; use fnv::FnvHashSet; use serde::{Deserialize, Serialize}; +use crate::error::invalid_data; use crate::expr::accessor::{StructAccessor, StructAccessorRef}; use crate::expr::{ BinaryExpression, Bind, Predicate, PredicateOperator, SetExpression, UnaryExpression, }; use crate::spec::{Datum, NestedField, NestedFieldRef, SchemaRef}; -use crate::{Error, ErrorKind}; /// Unbound term before binding to a schema. pub type Term = Reference; @@ -316,19 +316,12 @@ impl Bind for Reference { schema.field_by_name_case_insensitive(&self.name) }; - let field = field.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Field {} not found in schema", self.name), - ) - })?; - - let accessor = schema.accessor_by_field_id(field.id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Accessor for Field {} not found", self.name), - ) - })?; + let field = + field.ok_or_else(|| invalid_data!("Field {} not found in schema", self.name))?; + + let accessor = schema + .accessor_by_field_id(field.id) + .ok_or_else(|| invalid_data!("Accessor for Field {} not found", self.name))?; Ok(BoundReference::new( self.name.clone(), diff --git a/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs index 9fe8f216cb..6cb9f01c50 100644 --- a/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs @@ -17,10 +17,11 @@ use fnv::FnvHashSet; +use crate::Result; +use crate::error::invalid_data; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::{DataFile, Datum}; -use crate::{Error, ErrorKind, Result}; #[allow(dead_code)] const ROWS_MUST_MATCH: Result = Ok(true); @@ -159,10 +160,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { } fn not(&mut self, _inner: bool) -> Result { - Err(Error::new( - ErrorKind::DataInvalid, - "NOT should be rewritten", - )) + Err(invalid_data!("NOT should be rewritten")) } fn is_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { diff --git a/crates/iceberg/src/inspect/manifests.rs b/crates/iceberg/src/inspect/manifests.rs index 9e0f7f2240..e260375769 100644 --- a/crates/iceberg/src/inspect/manifests.rs +++ b/crates/iceberg/src/inspect/manifests.rs @@ -26,11 +26,12 @@ use arrow_array::types::{Int32Type, Int64Type}; use arrow_schema::{DataType, Field, Fields}; use futures::{StreamExt, stream}; +use crate::Result; use crate::arrow::schema_to_arrow_schema; +use crate::error::invalid_data; use crate::scan::ArrowRecordBatchStream; use crate::spec::{Datum, FieldSummary, ListType, NestedField, PrimitiveType, StructType, Type}; use crate::table::Table; -use crate::{Error, ErrorKind, Result}; /// Manifests table. pub struct ManifestsTable<'a> { @@ -185,12 +186,10 @@ impl<'a> ManifestsTable<'a> { .metadata() .partition_spec_by_id(manifest.partition_spec_id) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Partition spec {} for manifest {} is not in table metadata", - manifest.partition_spec_id, manifest.manifest_path - ), + invalid_data!( + "Partition spec {} for manifest {} is not in table metadata", + manifest.partition_spec_id, + manifest.manifest_path ) })?; let spec_struct = spec.partition_type(self.table.metadata().current_schema())?; diff --git a/crates/iceberg/src/io/storage/config/s3.rs b/crates/iceberg/src/io/storage/config/s3.rs index 664e8637b8..ae3bc4ba3b 100644 --- a/crates/iceberg/src/io/storage/config/s3.rs +++ b/crates/iceberg/src/io/storage/config/s3.rs @@ -24,8 +24,9 @@ use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use super::StorageConfig; +use crate::error::invalid_data; use crate::io::is_truthy; -use crate::{Error, ErrorKind, Result}; +use crate::{Error, Result}; /// S3 endpoint URL. pub const S3_ENDPOINT: &str = "s3.endpoint"; @@ -201,11 +202,8 @@ impl TryFrom<&StorageConfig> for S3Config { cfg.server_side_encryption_customer_key_md5 = props.get(S3_SSE_MD5).cloned(); } _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one of (custom, kms, s3, none)" - ), + return Err(invalid_data!( + "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one of (custom, kms, s3, none)" )); } } diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs index 836ea74d4e..d5ef0a4732 100644 --- a/crates/iceberg/src/io/storage/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -33,6 +33,7 @@ use futures::StreamExt; use futures::stream::BoxStream; use serde::{Deserialize, Serialize}; +use crate::error::invalid_data; use crate::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, StorageFactory, @@ -98,12 +99,8 @@ impl Storage for LocalFsStorage { async fn metadata(&self, path: &str) -> Result { let path = Self::normalize_path(path); - let metadata = fs::metadata(&path).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to get metadata for {}: {}", path.display(), e), - ) - })?; + let metadata = fs::metadata(&path) + .map_err(|e| invalid_data!("Failed to get metadata for {}: {}", path.display(), e))?; Ok(FileMetadata { size: metadata.len(), }) @@ -111,23 +108,15 @@ impl Storage for LocalFsStorage { async fn read(&self, path: &str) -> Result { let path = Self::normalize_path(path); - let content = fs::read(&path).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to read file {}: {}", path.display(), e), - ) - })?; + let content = fs::read(&path) + .map_err(|e| invalid_data!("Failed to read file {}: {}", path.display(), e))?; Ok(Bytes::from(content)) } async fn reader(&self, path: &str) -> Result> { let path = Self::normalize_path(path); - let file = fs::File::open(&path).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to open file {}: {}", path.display(), e), - ) - })?; + let file = fs::File::open(&path) + .map_err(|e| invalid_data!("Failed to open file {}: {}", path.display(), e))?; Ok(Box::new(LocalFsFileRead::new(file))) } @@ -242,21 +231,13 @@ impl FileRead for LocalFsFileRead { ) })?; - file.seek(SeekFrom::Start(range.start)).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to seek to position {}: {}", range.start, e), - ) - })?; + file.seek(SeekFrom::Start(range.start)) + .map_err(|e| invalid_data!("Failed to seek to position {}: {}", range.start, e))?; let len = (range.end - range.start) as usize; let mut buffer = vec![0u8; len]; - file.read_exact(&mut buffer).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to read {len} bytes: {e}"), - ) - })?; + file.read_exact(&mut buffer) + .map_err(|e| invalid_data!("Failed to read {len} bytes: {e}"))?; Ok(Bytes::from(buffer)) } @@ -283,7 +264,7 @@ impl FileWrite for LocalFsFileWrite { let file = self .file .as_mut() - .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "Cannot write to closed file"))?; + .ok_or_else(|| invalid_data!("Cannot write to closed file"))?; file.write_all(&bs).map_err(|e| { Error::new( @@ -299,7 +280,7 @@ impl FileWrite for LocalFsFileWrite { let file = self .file .take() - .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "File already closed"))?; + .ok_or_else(|| invalid_data!("File already closed"))?; file.sync_all() .map_err(|e| Error::new(ErrorKind::Unexpected, format!("Failed to sync file: {e}")))?; diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs index dc9ca10d6a..b39c741ea7 100644 --- a/crates/iceberg/src/io/storage/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -32,6 +32,7 @@ use futures::StreamExt; use futures::stream::BoxStream; use serde::{Deserialize, Serialize}; +use crate::error::invalid_data; use crate::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, StorageFactory, @@ -119,10 +120,7 @@ impl Storage for MemoryStorage { Some(bytes) => Ok(FileMetadata { size: bytes.len() as u64, }), - None => Err(Error::new( - ErrorKind::DataInvalid, - format!("File not found: {path}"), - )), + None => Err(invalid_data!("File not found: {path}")), } } @@ -136,10 +134,7 @@ impl Storage for MemoryStorage { })?; match data.get(&normalized) { Some(bytes) => Ok(bytes.clone()), - None => Err(Error::new( - ErrorKind::DataInvalid, - format!("File not found: {path}"), - )), + None => Err(invalid_data!("File not found: {path}")), } } @@ -153,10 +148,7 @@ impl Storage for MemoryStorage { })?; match data.get(&normalized) { Some(bytes) => Ok(Box::new(MemoryFileRead::new(bytes.clone()))), - None => Err(Error::new( - ErrorKind::DataInvalid, - format!("File not found: {path}"), - )), + None => Err(invalid_data!("File not found: {path}")), } } @@ -272,14 +264,11 @@ impl FileRead for MemoryFileRead { let end = range.end as usize; if start > self.data.len() || end > self.data.len() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Range {}..{} is out of bounds for data of length {}", - start, - end, - self.data.len() - ), + return Err(invalid_data!( + "Range {}..{} is out of bounds for data of length {}", + start, + end, + self.data.len() )); } @@ -316,10 +305,7 @@ impl MemoryFileWrite { impl FileWrite for MemoryFileWrite { async fn write(&mut self, bs: Bytes) -> Result<()> { if self.closed { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot write to closed file", - )); + return Err(invalid_data!("Cannot write to closed file")); } self.buffer.extend_from_slice(&bs); Ok(()) @@ -327,7 +313,7 @@ impl FileWrite for MemoryFileWrite { async fn close(&mut self) -> Result<()> { if self.closed { - return Err(Error::new(ErrorKind::DataInvalid, "File already closed")); + return Err(invalid_data!("File already closed")); } let mut data = self.data.write().map_err(|e| { diff --git a/crates/iceberg/src/partitioning.rs b/crates/iceberg/src/partitioning.rs index ae8a58c6ae..ef230ef9e5 100644 --- a/crates/iceberg/src/partitioning.rs +++ b/crates/iceberg/src/partitioning.rs @@ -20,6 +20,7 @@ use std::cmp::Reverse; use std::collections::{HashMap, HashSet}; +use crate::error::invalid_data; use crate::spec::{ NestedField, NestedFieldRef, PartitionField, PartitionSpec, Schema, StructType, Transform, Type, }; @@ -68,13 +69,10 @@ pub fn compute_unified_partition_type<'a>( // active_field_ids filter below, otherwise an unknown transform could be // silently skipped. if matches!(field.transform, Transform::Unknown) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Partition field '{}' uses an unknown transform whose result type \ + return Err(invalid_data!( + "Partition field '{}' uses an unknown transform whose result type \ cannot be determined", - field.name - ), + field.name )); } @@ -98,13 +96,11 @@ pub fn compute_unified_partition_type<'a>( // V1 tables do not guarantee field ids are unique across specs, so two // specs may define the same field id. They must be compatible. if !equivalent_ignoring_names(field, existing) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Conflicting partition fields for field id {field_id}: \ + return Err(invalid_data!( + "Conflicting partition fields for field id {field_id}: \ '{}' and '{}'", - field.name, existing.name - ), + field.name, + existing.name )); } diff --git a/crates/iceberg/src/puffin/metadata.rs b/crates/iceberg/src/puffin/metadata.rs index 8791cb4d8f..e237edf516 100644 --- a/crates/iceberg/src/puffin/metadata.rs +++ b/crates/iceberg/src/puffin/metadata.rs @@ -20,9 +20,10 @@ use std::collections::{HashMap, HashSet}; use bytes::Bytes; use serde::{Deserialize, Serialize}; +use crate::Result; use crate::compression::CompressionCodec; +use crate::error::invalid_data; use crate::io::FileRead; -use crate::{Error, ErrorKind, Result}; /// Human-readable identification of the application writing the file, along with its version. /// Example: "Trino version 381" @@ -119,9 +120,8 @@ impl Flag { if Flag::FooterPayloadCompressed.matches(byte_idx, bit_idx) { Ok(Flag::FooterPayloadCompressed) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("Unknown flag byte {byte_idx} and bit {bit_idx} combination"), + Err(invalid_data!( + "Unknown flag byte {byte_idx} and bit {bit_idx} combination" )) } } @@ -188,13 +188,10 @@ impl FileMetadata { if bytes == FileMetadata::MAGIC { Ok(()) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Bad magic value: {:?} should be {:?}", - bytes, - FileMetadata::MAGIC - ), + Err(invalid_data!( + "Bad magic value: {:?} should be {:?}", + bytes, + FileMetadata::MAGIC )) } } @@ -223,11 +220,8 @@ impl FileMetadata { let start = input_file_length .checked_sub(footer_length) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Footer length {footer_length} exceeds file length {input_file_length}" - ), + invalid_data!( + "Footer length {footer_length} exceeds file length {input_file_length}" ) })?; let end = input_file_length; @@ -243,9 +237,9 @@ impl FileMetadata { - FileMetadata::FOOTER_STRUCT_FLAGS_LENGTH as usize + byte_idx as usize; - let flag_byte = *footer_bytes.get(byte_offset).ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "Index range is out of bounds.") - })?; + let flag_byte = *footer_bytes + .get(byte_offset) + .ok_or_else(|| invalid_data!("Index range is out of bounds."))?; for bit_idx in 0..8 { if ((flag_byte >> bit_idx) & 1) != 0 { @@ -274,32 +268,26 @@ impl FileMetadata { FileMetadata::MAGIC_LENGTH as usize + usize::try_from(footer_payload_length)?; let footer_payload_bytes = footer_bytes .get(start_offset..end_offset) - .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "Index range is out of bounds."))?; + .ok_or_else(|| invalid_data!("Index range is out of bounds."))?; let decompressed_footer_payload_bytes = footer_compression_codec.decompress(footer_payload_bytes.into())?; - String::from_utf8(decompressed_footer_payload_bytes).map_err(|src| { - Error::new(ErrorKind::DataInvalid, "Footer is not a valid UTF-8 string") - .with_source(src) - }) + String::from_utf8(decompressed_footer_payload_bytes) + .map_err(|src| invalid_data!("Footer is not a valid UTF-8 string").with_source(src)) } fn from_json_str(string: &str) -> Result { - serde_json::from_str::(string).map_err(|src| { - Error::new(ErrorKind::DataInvalid, "Given string is not valid JSON").with_source(src) - }) + serde_json::from_str::(string) + .map_err(|src| invalid_data!("Given string is not valid JSON").with_source(src)) } /// Returns the file metadata about a Puffin file pub(crate) async fn read(file_read: &dyn FileRead, file_length: u64) -> Result { if file_length < FileMetadata::MIN_FILE_LENGTH { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "File length {} is too short to be a Puffin file, expected at least {} bytes", - file_length, - FileMetadata::MIN_FILE_LENGTH - ), + return Err(invalid_data!( + "File length {} is too short to be a Puffin file, expected at least {} bytes", + file_length, + FileMetadata::MIN_FILE_LENGTH )); } diff --git a/crates/iceberg/src/puffin/mod.rs b/crates/iceberg/src/puffin/mod.rs index 0e054cac51..998520cb8d 100644 --- a/crates/iceberg/src/puffin/mod.rs +++ b/crates/iceberg/src/puffin/mod.rs @@ -19,7 +19,8 @@ #![deny(missing_docs)] -use crate::{Error, ErrorKind, Result}; +use crate::Result; +use crate::error::invalid_data; mod blob; pub use blob::{APACHE_DATASKETCHES_THETA_V1, Blob, DELETION_VECTOR_V1}; @@ -31,15 +32,12 @@ pub use crate::compression::CompressionCodec; fn validate_puffin_compression(codec: CompressionCodec) -> Result<()> { match codec { CompressionCodec::None | CompressionCodec::Lz4 | CompressionCodec::Zstd(_) => Ok(()), - other => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Compression codec {} is not supported for Puffin files. Only {}, {}, and {} are supported.", - other.name(), - CompressionCodec::None.name(), - CompressionCodec::Lz4.name(), - CompressionCodec::zstd_default().name() - ), + other => Err(invalid_data!( + "Compression codec {} is not supported for Puffin files. Only {}, {}, and {} are supported.", + other.name(), + CompressionCodec::None.name(), + CompressionCodec::Lz4.name(), + CompressionCodec::zstd_default().name() )), } } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index fd833d9b38..7b39189bfb 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -35,6 +35,7 @@ pub use task::*; use crate::arrow::ArrowReaderBuilder; pub use crate::arrow::{ScanMetrics, ScanResult}; use crate::delete_file_index::DeleteFileIndex; +use crate::error::invalid_data; use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator; use crate::expr::{Bind, BoundPredicate, Predicate}; use crate::io::FileIO; @@ -78,10 +79,7 @@ fn collect_scan_field_ids( } let field_id = resolve_field_id(schema, column_name, case_sensitive).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Column {column_name} not found in table. Schema: {schema}"), - ) + invalid_data!("Column {column_name} not found in table. Schema: {schema}") })?; schema @@ -289,12 +287,7 @@ impl<'a> TableScanBuilder<'a> { .table .metadata() .snapshot_by_id(snapshot_id) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Snapshot with id {snapshot_id} not found"), - ) - })? + .ok_or_else(|| invalid_data!("Snapshot with id {snapshot_id} not found"))? .clone(), None => { let Some(current_snapshot_id) = self.table.metadata().current_snapshot() else { diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 531f405567..96a322e3d4 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -21,12 +21,13 @@ use futures::stream::BoxStream; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; +use crate::Result; +use crate::error::invalid_data; use crate::expr::BoundPredicate; use crate::spec::{ DataContentType, DataFileFormat, ManifestEntryRef, NameMapping, PartitionSpec, Schema, SchemaRef, SortOrderRef, Struct, StructType, }; -use crate::{Error, ErrorKind, Result}; /// A stream of [`FileScanTask`]. pub type FileScanTaskStream = BoxStream<'static, Result>; @@ -261,25 +262,20 @@ impl FileScanTask { match (self.partition.as_ref(), self.partition_spec.as_deref()) { (None, None) => Ok(()), (None, Some(partition_spec)) if partition_spec.is_unpartitioned() => Ok(()), - (None, Some(_)) => Err(Error::new( - ErrorKind::DataInvalid, - "FileScanTask with a partitioned spec requires partition values", + (None, Some(_)) => Err(invalid_data!( + "FileScanTask with a partitioned spec requires partition values" )), (Some(partition), None) if partition.fields().is_empty() => Ok(()), - (Some(_), None) => Err(Error::new( - ErrorKind::DataInvalid, - "Non-empty FileScanTask partition requires a partition spec", + (Some(_), None) => Err(invalid_data!( + "Non-empty FileScanTask partition requires a partition spec" )), (Some(partition), Some(partition_spec)) if partition.fields().len() != partition_spec.fields().len() => { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "FileScanTask partition has {} fields but partition spec has {} fields", - partition.fields().len(), - partition_spec.fields().len() - ), + Err(invalid_data!( + "FileScanTask partition has {} fields but partition spec has {} fields", + partition.fields().len(), + partition_spec.fields().len() )) } (Some(_), Some(partition_spec)) => { @@ -400,12 +396,13 @@ mod _serde { use serde::{Deserialize, Serialize}; use super::{FileScanTask, FileScanTaskDeleteFile}; + use crate::error::invalid_data; use crate::expr::BoundPredicate; use crate::spec::{ DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral, SchemaRef, SortOrderRef, StructType, Type, }; - use crate::{Error, ErrorKind, Result}; + use crate::{Error, Result}; #[derive(Deserialize)] pub(super) struct FileScanTaskSerde { @@ -543,10 +540,7 @@ mod _serde { partition_type(value.partition_spec.as_deref(), &value.schema)?; match partition.try_into(&partition_type)? { Some(Literal::Struct(partition)) => Ok(partition), - _ => Err(Error::new( - ErrorKind::DataInvalid, - "FileScanTask partition must be a struct", - )), + _ => Err(invalid_data!("FileScanTask partition must be a struct")), } }) .transpose()?; diff --git a/crates/iceberg/src/spec/manifest/_serde.rs b/crates/iceberg/src/spec/manifest/_serde.rs index 5bc0bf9372..336c883f1a 100644 --- a/crates/iceberg/src/spec/manifest/_serde.rs +++ b/crates/iceberg/src/spec/manifest/_serde.rs @@ -21,8 +21,9 @@ use serde_derive::{Deserialize, Serialize}; use serde_with::serde_as; use super::{Datum, ManifestEntry, Schema, Struct}; +use crate::error::invalid_data; use crate::spec::{FormatVersion, Literal, RawLiteral, Type}; -use crate::{Error, ErrorKind, metadata_columns}; +use crate::{Error, metadata_columns}; #[derive(Serialize, Deserialize)] pub(super) struct ManifestEntryV2 { @@ -173,10 +174,7 @@ impl DataFileSerde { if let Literal::Struct(v) = v { Ok(v) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "partition value is not a struct", - )) + Err(invalid_data!("partition value is not a struct")) } }) .transpose()? @@ -251,12 +249,7 @@ fn parse_bytes_entry(v: Vec, schema: &Schema) -> Result for DataContentType { 0 => Ok(DataContentType::Data), 1 => Ok(DataContentType::PositionDeletes), 2 => Ok(DataContentType::EqualityDeletes), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("data content type {v} is invalid"), - )), + _ => Err(invalid_data!("data content type {v} is invalid")), } } } @@ -400,10 +397,7 @@ impl FromStr for DataFileFormat { "orc" => Ok(Self::Orc), "parquet" => Ok(Self::Parquet), "puffin" => Ok(Self::Puffin), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Unsupported data file format: {s}"), - )), + _ => Err(invalid_data!("Unsupported data file format: {s}")), } } } diff --git a/crates/iceberg/src/spec/manifest/entry.rs b/crates/iceberg/src/spec/manifest/entry.rs index e8fe0f223a..555d86a25c 100644 --- a/crates/iceberg/src/spec/manifest/entry.rs +++ b/crates/iceberg/src/spec/manifest/entry.rs @@ -21,13 +21,13 @@ use apache_avro::Schema as AvroSchema; use once_cell::sync::Lazy; use typed_builder::TypedBuilder; +use crate::Error; use crate::avro::schema_to_avro_schema; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::{ DataContentType, DataFile, INITIAL_SEQUENCE_NUMBER, ListType, Literal, ManifestFile, MapType, NestedField, NestedFieldRef, PrimitiveLiteral, PrimitiveType, Schema, StructType, Type, }; -use crate::{Error, ErrorKind}; /// Reference to [`ManifestEntry`]. pub type ManifestEntryRef = Arc; @@ -170,10 +170,7 @@ impl TryFrom for ManifestStatus { 0 => Ok(ManifestStatus::Existing), 1 => Ok(ManifestStatus::Added), 2 => Ok(ManifestStatus::Deleted), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("manifest status {v} is invalid"), - )), + _ => Err(invalid_data!("manifest status {v} is invalid")), } } } diff --git a/crates/iceberg/src/spec/manifest/metadata.rs b/crates/iceberg/src/spec/manifest/metadata.rs index 25e4ae7e06..b06122215e 100644 --- a/crates/iceberg/src/spec/manifest/metadata.rs +++ b/crates/iceberg/src/spec/manifest/metadata.rs @@ -21,9 +21,8 @@ use std::sync::Arc; use typed_builder::TypedBuilder; use super::{FormatVersion, ManifestContentType, PartitionSpec, Schema}; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::{PartitionField, SchemaId, SchemaRef}; -use crate::{Error, ErrorKind}; /// Meta data of a manifest that is stored in the key-value metadata of the Avro file #[derive(Debug, PartialEq, Clone, Eq, TypedBuilder)] @@ -46,28 +45,17 @@ impl ManifestMetadata { pub fn parse(meta: &HashMap>) -> Result { let schema = Arc::new({ let bs = meta.get("schema").ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "schema is required in manifest metadata but not found", - ) + invalid_data!("schema is required in manifest metadata but not found") })?; serde_json::from_slice::(bs).map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse schema in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse schema in manifest metadata").with_source(err) })? }); let schema_id: i32 = meta .get("schema-id") .map(|bs| { String::from_utf8_lossy(bs).parse().map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse schema id in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse schema id in manifest metadata").with_source(err) }) }) .transpose()? @@ -75,28 +63,19 @@ impl ManifestMetadata { let partition_spec = { let fields = { let bs = meta.get("partition-spec").ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "partition-spec is required in manifest metadata but not found", - ) + invalid_data!("partition-spec is required in manifest metadata but not found") })?; serde_json::from_slice::>(bs).map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse partition spec in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse partition spec in manifest metadata") + .with_source(err) })? }; let spec_id = meta .get("partition-spec-id") .map(|bs| { String::from_utf8_lossy(bs).parse().map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse partition spec id in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse partition spec id in manifest metadata") + .with_source(err) }) }) .transpose()? @@ -108,11 +87,7 @@ impl ManifestMetadata { }; let format_version = if let Some(bs) = meta.get("format-version") { serde_json::from_slice::(bs).map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse format version in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse format version in manifest metadata").with_source(err) })? } else { FormatVersion::V1 diff --git a/crates/iceberg/src/spec/manifest/mod.rs b/crates/iceberg/src/spec/manifest/mod.rs index f05eeae774..3bd4dc8ad8 100644 --- a/crates/iceberg/src/spec/manifest/mod.rs +++ b/crates/iceberg/src/spec/manifest/mod.rs @@ -35,8 +35,7 @@ use super::{ Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType, Schema, Struct, Type, UNASSIGNED_SEQUENCE_NUMBER, }; -use crate::error::Result; -use crate::{Error, ErrorKind}; +use crate::error::{Result, invalid_data}; /// A manifest contains metadata and a list of entries. #[derive(Debug, PartialEq, Eq, Clone)] @@ -135,13 +134,8 @@ pub fn serialize_data_file_to_json( ) -> Result { let partition_struct_type = Type::Struct(partition_type.clone()); let serde = _serde::DataFileSerde::try_from(data_file, &partition_struct_type, format_version)?; - serde_json::to_string(&serde).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to serialize DataFile to JSON!".to_string(), - ) - .with_source(e) - }) + serde_json::to_string(&serde) + .map_err(|e| invalid_data!("Failed to serialize DataFile to JSON!").with_source(e)) } /// Deserialize a DataFile from a JSON string. @@ -151,13 +145,8 @@ pub fn deserialize_data_file_from_json( partition_type: &super::StructType, schema: &Schema, ) -> Result { - let serde = serde_json::from_str::<_serde::DataFileSerde>(json).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to deserialize JSON to DataFile!".to_string(), - ) - .with_source(e) - })?; + let serde = serde_json::from_str::<_serde::DataFileSerde>(json) + .map_err(|e| invalid_data!("Failed to deserialize JSON to DataFile!").with_source(e))?; let partition_struct_type = Type::Struct(partition_type.clone()); serde.try_into(partition_spec_id, &partition_struct_type, schema) diff --git a/crates/iceberg/src/spec/manifest/reader.rs b/crates/iceberg/src/spec/manifest/reader.rs index 499ff8fc21..e7044a56bc 100644 --- a/crates/iceberg/src/spec/manifest/reader.rs +++ b/crates/iceberg/src/spec/manifest/reader.rs @@ -17,10 +17,9 @@ use super::Manifest; use crate::encryption::{EncryptedInputFile, StandardKeyMetadata}; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::io::FileIO; use crate::spec::{ManifestContentType, ManifestEntry, ManifestFile}; -use crate::{Error, ErrorKind}; /// Reads a manifest file referenced by a manifest list entry, transparently /// decrypting it when the entry records key metadata. @@ -103,10 +102,7 @@ impl ManifestReader { }; let mut next_row_id = i64::try_from(manifest_first_row_id).map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid first_row_id: {manifest_first_row_id} (exceeds i64::MAX)"), - ) + invalid_data!("Invalid first_row_id: {manifest_first_row_id} (exceeds i64::MAX)") })?; for entry in entries { @@ -119,13 +115,8 @@ impl ManifestReader { entry.data_file.first_row_id = Some(file_first_row_id); let record_count = entry.data_file.record_count; next_row_id = file_first_row_id.checked_add_unsigned(record_count).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Row ID overflow assigning first_row_id in {}. File first_row_id: {file_first_row_id}, record count: {record_count}", - manifest_file.manifest_path - ), - ) + invalid_data!("Row ID overflow assigning first_row_id in {}. File first_row_id: {file_first_row_id}, record count: {record_count}", + manifest_file.manifest_path) })?; } } @@ -140,6 +131,7 @@ mod tests { use std::sync::Arc; use super::*; + use crate::ErrorKind; use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata}; use crate::io::FileIO; use crate::spec::{ diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index 7256692950..b80567a683 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -29,7 +29,7 @@ use super::{ UNASSIGNED_SEQUENCE_NUMBER, }; use crate::encryption::EncryptedOutputFile; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::io::{FileWrite, OutputFile}; use crate::spec::manifest::_serde::{ManifestEntryV1, ManifestEntryV2}; use crate::spec::manifest::{manifest_schema_v1, manifest_schema_v2}; @@ -37,7 +37,6 @@ use crate::spec::{ DataContentType, DataFile, FieldSummary, ManifestEntry, ManifestFile, ManifestMetadata, ManifestStatus, PrimitiveLiteral, SchemaRef, StructType, Type, }; -use crate::{Error, ErrorKind}; /// Placeholder for snapshot ID. The field with this value must be replaced /// with the actual snapshot ID before it is committed. @@ -268,13 +267,10 @@ impl ManifestWriter { match self.metadata.content { ManifestContentType::Data => { if data_file.content != DataContentType::Data { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Date file at path {} with manifest content type `data`, should have DataContentType `Data`, but has `{:?}`", - data_file.file_path(), - data_file.content - ), + return Err(invalid_data!( + "Date file at path {} with manifest content type `data`, should have DataContentType `Data`, but has `{:?}`", + data_file.file_path(), + data_file.content )); } } @@ -282,13 +278,10 @@ impl ManifestWriter { if data_file.content != DataContentType::EqualityDeletes && data_file.content != DataContentType::PositionDeletes { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Date file at path {} with manifest content type `deletes`, should have DataContentType `Data`, but has `{:?}`", - data_file.file_path(), - data_file.content - ), + return Err(invalid_data!( + "Date file at path {} with manifest content type `deletes`, should have DataContentType `Data`, but has `{:?}`", + data_file.file_path(), + data_file.content )); } } @@ -408,9 +401,8 @@ impl ManifestWriter { if (entry.status == ManifestStatus::Deleted || entry.status == ManifestStatus::Existing) && (entry.sequence_number.is_none() || entry.file_sequence_number.is_none()) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Manifest entry with status Existing or Deleted should have sequence number", + return Err(invalid_data!( + "Manifest entry with status Existing or Deleted should have sequence number" )); } @@ -457,10 +449,8 @@ impl ManifestWriter { let mut avro_writer = AvroWriter::new(&avro_schema, Vec::new()); avro_writer.add_user_metadata( "schema".to_string(), - to_vec(table_schema).map_err(|err| { - Error::new(ErrorKind::DataInvalid, "Fail to serialize table schema") - .with_source(err) - })?, + to_vec(table_schema) + .map_err(|err| invalid_data!("Fail to serialize table schema").with_source(err))?, )?; avro_writer.add_user_metadata( "schema-id".to_string(), @@ -469,8 +459,7 @@ impl ManifestWriter { avro_writer.add_user_metadata( "partition-spec".to_string(), to_vec(&self.metadata.partition_spec.fields()).map_err(|err| { - Error::new(ErrorKind::DataInvalid, "Fail to serialize partition spec") - .with_source(err) + invalid_data!("Fail to serialize partition spec").with_source(err) })?, )?; avro_writer.add_user_metadata( @@ -562,10 +551,7 @@ impl PartitionFieldStats { return Ok(()); }; if !self.partition_type.compatible(&value) { - return Err(Error::new( - ErrorKind::DataInvalid, - "value is not compatible with type", - )); + return Err(invalid_data!("value is not compatible with type")); } let value = Datum::new(self.partition_type.clone(), value); diff --git a/crates/iceberg/src/spec/manifest_list/_serde.rs b/crates/iceberg/src/spec/manifest_list/_serde.rs index 2a10c39ea3..3528dec7cf 100644 --- a/crates/iceberg/src/spec/manifest_list/_serde.rs +++ b/crates/iceberg/src/spec/manifest_list/_serde.rs @@ -24,7 +24,7 @@ use serde_derive::{Deserialize, Serialize}; use super::ManifestFile; use crate::Error; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::FieldSummary; #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -319,56 +319,32 @@ impl TryFrom for ManifestFileV3 { added_files_count: value .added_files_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "added_data_files_count in ManifestFileV3 is required", - ) + invalid_data!("added_data_files_count in ManifestFileV3 is required") })? .try_into()?, existing_files_count: value .existing_files_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "existing_data_files_count in ManifestFileV3 is required", - ) + invalid_data!("existing_data_files_count in ManifestFileV3 is required") })? .try_into()?, deleted_files_count: value .deleted_files_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "deleted_data_files_count in ManifestFileV3 is required", - ) + invalid_data!("deleted_data_files_count in ManifestFileV3 is required") })? .try_into()?, added_rows_count: value .added_rows_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "added_rows_count in ManifestFileV3 is required", - ) - })? + .ok_or_else(|| invalid_data!("added_rows_count in ManifestFileV3 is required"))? .try_into()?, existing_rows_count: value .existing_rows_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "existing_rows_count in ManifestFileV3 is required", - ) - })? + .ok_or_else(|| invalid_data!("existing_rows_count in ManifestFileV3 is required"))? .try_into()?, deleted_rows_count: value .deleted_rows_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "deleted_rows_count in ManifestFileV3 is required", - ) - })? + .ok_or_else(|| invalid_data!("deleted_rows_count in ManifestFileV3 is required"))? .try_into()?, partitions: value.partitions, key_metadata, @@ -393,55 +369,37 @@ impl TryFrom for ManifestFileV2 { added_files_count: value .added_files_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "added_data_files_count in ManifestFileV2 should be require", - ) + invalid_data!("added_data_files_count in ManifestFileV2 should be require") })? .try_into()?, existing_files_count: value .existing_files_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "existing_data_files_count in ManifestFileV2 should be require", - ) + invalid_data!("existing_data_files_count in ManifestFileV2 should be require") })? .try_into()?, deleted_files_count: value .deleted_files_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "deleted_data_files_count in ManifestFileV2 should be require", - ) + invalid_data!("deleted_data_files_count in ManifestFileV2 should be require") })? .try_into()?, added_rows_count: value .added_rows_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "added_rows_count in ManifestFileV2 should be require", - ) + invalid_data!("added_rows_count in ManifestFileV2 should be require") })? .try_into()?, existing_rows_count: value .existing_rows_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "existing_rows_count in ManifestFileV2 should be require", - ) + invalid_data!("existing_rows_count in ManifestFileV2 should be require") })? .try_into()?, deleted_rows_count: value .deleted_rows_count .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "deleted_rows_count in ManifestFileV2 should be require", - ) + invalid_data!("deleted_rows_count in ManifestFileV2 should be require") })? .try_into()?, partitions: value.partitions, diff --git a/crates/iceberg/src/spec/manifest_list/manifest_file.rs b/crates/iceberg/src/spec/manifest_list/manifest_file.rs index 567e78e3a5..341748a416 100644 --- a/crates/iceberg/src/spec/manifest_list/manifest_file.rs +++ b/crates/iceberg/src/spec/manifest_list/manifest_file.rs @@ -20,8 +20,8 @@ use std::str::FromStr; use serde_derive::{Deserialize, Serialize}; use super::ByteBuf; -use crate::error::Result; -use crate::{Error, ErrorKind}; +use crate::Error; +use crate::error::{Result, invalid_data}; /// Entry in a manifest list. #[derive(Debug, PartialEq, Clone, Eq, Hash)] @@ -139,10 +139,7 @@ impl FromStr for ManifestContentType { match s { "data" => Ok(ManifestContentType::Data), "deletes" => Ok(ManifestContentType::Deletes), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid manifest content type: {s}"), - )), + _ => Err(invalid_data!("Invalid manifest content type: {s}")), } } } @@ -163,9 +160,8 @@ impl TryFrom for ManifestContentType { match value { 0 => Ok(ManifestContentType::Data), 1 => Ok(ManifestContentType::Deletes), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid manifest content type. Expected 0 or 1, got {value}"), + _ => Err(invalid_data!( + "Invalid manifest content type. Expected 0 or 1, got {value}" )), } } diff --git a/crates/iceberg/src/spec/manifest_list/writer.rs b/crates/iceberg/src/spec/manifest_list/writer.rs index 710471780b..24613770d2 100644 --- a/crates/iceberg/src/spec/manifest_list/writer.rs +++ b/crates/iceberg/src/spec/manifest_list/writer.rs @@ -25,7 +25,7 @@ use super::_const_schema::{ }; use super::_serde::{ManifestFileV1, ManifestFileV2, ManifestFileV3}; use super::{FormatVersion, ManifestContentType, ManifestFile, UNASSIGNED_SEQUENCE_NUMBER}; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::io::FileWrite; use crate::{Error, ErrorKind}; @@ -208,12 +208,9 @@ impl ManifestListWriter { fn assign_sequence_numbers(&self, manifest: &mut ManifestFile) -> Result<()> { if manifest.sequence_number == UNASSIGNED_SEQUENCE_NUMBER { if manifest.added_snapshot_id != self.snapshot_id { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Found unassigned sequence number for a manifest from snapshot {}.", - manifest.added_snapshot_id - ), + return Err(invalid_data!( + "Found unassigned sequence number for a manifest from snapshot {}.", + manifest.added_snapshot_id )); } manifest.sequence_number = self.sequence_number; @@ -221,12 +218,9 @@ impl ManifestListWriter { if manifest.min_sequence_number == UNASSIGNED_SEQUENCE_NUMBER { if manifest.added_snapshot_id != self.snapshot_id { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Found unassigned sequence number for a manifest from snapshot {}.", - manifest.added_snapshot_id - ), + return Err(invalid_data!( + "Found unassigned sequence number for a manifest from snapshot {}.", + manifest.added_snapshot_id )); } manifest.min_sequence_number = self.sequence_number; @@ -265,13 +259,10 @@ impl ManifestListWriter { .checked_add(existing_rows_count) .and_then(|sum| sum.checked_add(added_rows_count)) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Row ID overflow when computing next row ID for Manifest {}. Next Row ID: {writer_next_row_id}, Existing Rows Count: {existing_rows_count}, Added Rows Count: {added_rows_count}", manifest.manifest_path - ), - ) + ) }).map(Some)?; } (None, None) => { @@ -291,22 +282,16 @@ impl ManifestListWriter { fn require_row_counts_in_manifest(manifest: &ManifestFile) -> Result<(u64, u64)> { let existing_rows_count = manifest.existing_rows_count.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Cannot include a Manifest without existing-rows-count to a table with row lineage enabled. Manifest path: {}", manifest.manifest_path, - ), - ) + ) })?; let added_rows_count = manifest.added_rows_count.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( + invalid_data!( "Cannot include a Manifest without added-rows-count to a table with row lineage enabled. Manifest path: {}", manifest.manifest_path, - ), - ) + ) })?; Ok((existing_rows_count, added_rows_count)) } diff --git a/crates/iceberg/src/spec/name_mapping/mod.rs b/crates/iceberg/src/spec/name_mapping/mod.rs index a87e829ad0..95b8642fa6 100644 --- a/crates/iceberg/src/spec/name_mapping/mod.rs +++ b/crates/iceberg/src/spec/name_mapping/mod.rs @@ -23,7 +23,8 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; -use crate::{Error, ErrorKind, Result}; +use crate::error::invalid_data; +use crate::{Error, Result}; /// Iceberg fallback field name to ID mapping. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] @@ -50,11 +51,7 @@ impl FromStr for NameMapping { /// Parses a [`NameMapping`] from its JSON representation. fn from_str(value: &str) -> Result { serde_json::from_str(value).map_err(|error| { - Error::new( - ErrorKind::DataInvalid, - "Failed to parse value as a NameMapping", - ) - .with_source(error) + invalid_data!("Failed to parse value as a NameMapping").with_source(error) }) } } diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index c79eaae639..3999c0d010 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -26,8 +26,9 @@ use typed_builder::TypedBuilder; use super::transform::Transform; use super::{NestedField, PrimitiveType, Schema, SchemaRef, StructType}; +use crate::Result; +use crate::error::invalid_data; use crate::spec::Struct; -use crate::{Error, ErrorKind, Result}; pub(crate) const UNPARTITIONED_LAST_ASSIGNED_ID: i32 = 999; pub(crate) const DEFAULT_PARTITION_SPEC_ID: i32 = 0; @@ -487,12 +488,9 @@ impl PartitionSpecBuilder { .schema .field_by_name(source_name.as_ref()) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot find source column with name: {} in schema", - source_name.as_ref() - ), + invalid_data!( + "Cannot find source column with name: {} in schema", + source_name.as_ref() ) })? .id; @@ -558,12 +556,8 @@ impl PartitionSpecBuilder { .collect::>(); fn _check_add_1(prev: i32) -> Result { - prev.checked_add(1).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot assign more partition ids. Overflow.", - ) - }) + prev.checked_add(1) + .ok_or_else(|| invalid_data!("Cannot assign more partition ids. Overflow.")) } let mut bound_fields = Vec::with_capacity(fields.len()); @@ -604,21 +598,17 @@ impl PartitionSpecBuilder { if schema_collision.id == field.source_id { Ok(()) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot create identity partition sourced from different field in schema. Field name '{}' has id `{}` in schema but partition source id is `{}`", - field.name, schema_collision.id, field.source_id - ), + Err(invalid_data!( + "Cannot create identity partition sourced from different field in schema. Field name '{}' has id `{}` in schema but partition source id is `{}`", + field.name, + schema_collision.id, + field.source_id )) } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot create partition with name: '{}' that conflicts with schema field and is not an identity transform.", - field.name - ), + Err(invalid_data!( + "Cannot create partition with name: '{}' that conflicts with schema field and is not an identity transform.", + field.name )) } } @@ -630,23 +620,17 @@ impl PartitionSpecBuilder { /// in the schema. Implicitly also checks if the source field exists in the schema. fn check_transform_compatibility(field: &UnboundPartitionField, schema: &Schema) -> Result<()> { let schema_field = schema.field_by_id(field.source_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot find partition source field with id `{}` in schema", - field.source_id - ), + invalid_data!( + "Cannot find partition source field with id `{}` in schema", + field.source_id ) })?; if field.transform != Transform::Void { if !schema_field.field_type.is_primitive() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot partition by non-primitive source field: '{}'.", - schema_field.field_type - ), + return Err(invalid_data!( + "Cannot partition by non-primitive source field: '{}'.", + schema_field.field_type )); } @@ -655,13 +639,10 @@ impl PartitionSpecBuilder { .result_type(&schema_field.field_type) .is_err() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid source type: '{}' for transform: '{}'.", - schema_field.field_type, - field.transform.dedup_name() - ), + return Err(invalid_data!( + "Invalid source type: '{}' for transform: '{}'.", + schema_field.field_type, + field.transform.dedup_name() )); } } @@ -675,16 +656,12 @@ trait CorePartitionSpecValidator { /// Ensure that the partition name is unique among the partition fields and is not empty. fn check_name_set_and_unique(&self, name: &str) -> Result<()> { if name.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot use empty partition name", - )); + return Err(invalid_data!("Cannot use empty partition name")); } if self.fields().iter().any(|f| f.name == name) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot use partition name more than once: {name}"), + return Err(invalid_data!( + "Cannot use partition name more than once: {name}" )); } Ok(()) @@ -697,14 +674,11 @@ trait CorePartitionSpecValidator { }); if let Some(collision) = collision { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add redundant partition with source id `{}` and transform `{}`. A partition with the same source id and transform already exists with name `{}`", - source_id, - transform.dedup_name(), - collision.name - ), + Err(invalid_data!( + "Cannot add redundant partition with source id `{}` and transform `{}`. A partition with the same source id and transform already exists with name `{}`", + source_id, + transform.dedup_name(), + collision.name )) } else { Ok(()) @@ -714,9 +688,8 @@ trait CorePartitionSpecValidator { /// Check field / partition_id unique within the partition spec if set fn check_partition_id_unique(&self, field_id: i32) -> Result<()> { if self.fields().iter().any(|f| f.field_id == Some(field_id)) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot use field id more than once in one PartitionSpec: {field_id}"), + return Err(invalid_data!( + "Cannot use field id more than once in one PartitionSpec: {field_id}" )); } @@ -741,6 +714,7 @@ impl CorePartitionSpecValidator for UnboundPartitionSpecBuilder { #[cfg(test)] mod tests { use super::*; + use crate::ErrorKind; use crate::spec::{Literal, PrimitiveType, Type}; #[test] diff --git a/crates/iceberg/src/spec/schema/id_reassigner.rs b/crates/iceberg/src/spec/schema/id_reassigner.rs index d817ccbd1f..0c9b4de3f9 100644 --- a/crates/iceberg/src/spec/schema/id_reassigner.rs +++ b/crates/iceberg/src/spec/schema/id_reassigner.rs @@ -17,6 +17,7 @@ use super::utils::try_insert_field; use super::*; +use crate::error::invalid_data; pub struct ReassignFieldIds { next_field_id: i32, @@ -107,12 +108,10 @@ impl ReassignFieldIds { } fn increase_next_field_id(&mut self) -> Result<()> { - self.next_field_id = self.next_field_id.checked_add(1).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Field ID overflowed, cannot add more fields", - ) - })?; + self.next_field_id = self + .next_field_id + .checked_add(1) + .ok_or_else(|| invalid_data!("Field ID overflowed, cannot add more fields"))?; Ok(()) } @@ -120,12 +119,10 @@ impl ReassignFieldIds { field_ids .into_iter() .map(|id| { - self.old_to_new_id.get(&id).copied().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Identifier Field ID {id} not found"), - ) - }) + self.old_to_new_id + .get(&id) + .copied() + .ok_or_else(|| invalid_data!("Identifier Field ID {id} not found")) }) .collect() } @@ -140,12 +137,7 @@ impl ReassignFieldIds { self.old_to_new_id .get(&id) .copied() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Field with id {id} for alias {name} not found"), - ) - }) + .ok_or_else(|| invalid_data!("Field with id {id} for alias {name} not found")) .map(|new_id| (name, new_id)) }) .collect() diff --git a/crates/iceberg/src/spec/schema/index.rs b/crates/iceberg/src/spec/schema/index.rs index e4358e9ef9..eb2f990a36 100644 --- a/crates/iceberg/src/spec/schema/index.rs +++ b/crates/iceberg/src/spec/schema/index.rs @@ -17,6 +17,7 @@ use super::utils::try_insert_field; use super::*; +use crate::error::invalid_data; use crate::spec::VariantType; /// Creates a field id to field map. @@ -183,11 +184,8 @@ impl IndexByName { .chain(vec![name]) .join("."); if let Some(existing_field_id) = self.name_to_id.get(full_name.as_str()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}" - ), + return Err(invalid_data!( + "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}" )); } else { self.name_to_id.insert(full_name, field_id); diff --git a/crates/iceberg/src/spec/schema/mod.rs b/crates/iceberg/src/spec/schema/mod.rs index a0197b82c9..c6dcf07079 100644 --- a/crates/iceberg/src/spec/schema/mod.rs +++ b/crates/iceberg/src/spec/schema/mod.rs @@ -37,7 +37,7 @@ use self::id_reassigner::ReassignFieldIds; use self::index::{IndexByName, index_by_id, index_parents}; pub use self::prune_columns::prune_columns; use super::NestedField; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::expr::accessor::StructAccessor; use crate::spec::FormatVersion; use crate::spec::datatypes::{ @@ -258,11 +258,8 @@ impl SchemaBuilder { let id_to_parent = index_parents(r#struct)?; for identifier_field_id in identifier_field_ids { let field = id_to_field.get(&identifier_field_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add identifier field {identifier_field_id}: field does not exist" - ), + invalid_data!( + "Cannot add identifier field {identifier_field_id}: field does not exist" ) })?; ensure_data_valid!( @@ -277,12 +274,9 @@ impl SchemaBuilder { field.name ); } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add field {} as an identifier field: not a primitive type field", - field.name - ), + return Err(invalid_data!( + "Cannot add field {} as an identifier field: not a primitive type field", + field.name )); } @@ -504,9 +498,8 @@ impl Schema { .sorted_by_key(|(id, _)| *id) .map(|(_, msg)| msg) .join("\n- "); - Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid schema for {format_version}:\n- {message}"), + Err(invalid_data!( + "Invalid schema for {format_version}:\n- {message}" )) } } diff --git a/crates/iceberg/src/spec/schema/prune_columns.rs b/crates/iceberg/src/spec/schema/prune_columns.rs index e80d86221d..edfbb5fa1d 100644 --- a/crates/iceberg/src/spec/schema/prune_columns.rs +++ b/crates/iceberg/src/spec/schema/prune_columns.rs @@ -16,6 +16,7 @@ // under the License. use super::*; +use crate::error::invalid_data; use crate::spec::VariantType; struct PruneColumn { @@ -117,10 +118,8 @@ impl SchemaVisitor for PruneColumn { } else if !field.field_type.is_nested() { Ok(Some(*field.field_type.clone())) } else { - Err(Error::new( - ErrorKind::DataInvalid, + Err(invalid_data!( "Can't project list or map field directly when not selecting full type." - .to_string(), ) .with_context("field_id", field.id.to_string()) .with_context("field_type", field.field_type.to_string())) @@ -182,12 +181,10 @@ impl SchemaVisitor for PruneColumn { } else if list.element_field.field_type.is_primitive() { Ok(Some(Type::List(list.clone()))) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot explicitly project List or Map types, List element {} of type {} was selected", - list.element_field.id, list.element_field.field_type - ), + Err(invalid_data!( + "Cannot explicitly project List or Map types, List element {} of type {} was selected", + list.element_field.id, + list.element_field.field_type )) } } else if let Some(result) = value { @@ -216,12 +213,10 @@ impl SchemaVisitor for PruneColumn { } else if map.value_field.field_type.is_primitive() { Ok(Some(Type::Map(map.clone()))) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot explicitly project List or Map types, Map value {} of type {} was selected", - map.value_field.id, map.value_field.field_type - ), + Err(invalid_data!( + "Cannot explicitly project List or Map types, Map value {} of type {} was selected", + map.value_field.id, + map.value_field.field_type )) } } else if let Some(value_result) = value { diff --git a/crates/iceberg/src/spec/schema/utils.rs b/crates/iceberg/src/spec/schema/utils.rs index 34ee39b099..26f49b23da 100644 --- a/crates/iceberg/src/spec/schema/utils.rs +++ b/crates/iceberg/src/spec/schema/utils.rs @@ -17,15 +17,15 @@ use std::collections::HashMap; -use crate::{Error, ErrorKind, Result}; +use crate::Result; +use crate::error::invalid_data; pub fn try_insert_field(map: &mut HashMap, field_id: i32, value: V) -> Result<()> { map.insert(field_id, value).map_or_else( || Ok(()), |_| { - Err(Error::new( - ErrorKind::DataInvalid, - format!("Found duplicate 'field.id' {field_id}. Field ids must be unique."), + Err(invalid_data!( + "Found duplicate 'field.id' {field_id}. Field ids must be unique." )) }, ) diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 9b0e11cf10..bbd53a0c32 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -25,9 +25,8 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; -use crate::error::{Result, timestamp_ms_to_utc}; +use crate::error::{Result, invalid_data, timestamp_ms_to_utc}; use crate::spec::{SchemaId, SchemaRef, TableMetadata}; -use crate::{Error, ErrorKind}; /// The ref name of the main branch of the table. pub const MAIN_BRANCH: &str = "main"; @@ -172,12 +171,7 @@ impl Snapshot { Ok(match self.schema_id() { Some(schema_id) => table_metadata .schema_by_id(schema_id) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Schema with id {schema_id} not found"), - ) - })? + .ok_or_else(|| invalid_data!("Schema with id {schema_id} not found"))? .clone(), None => table_metadata.current_schema().clone(), }) @@ -234,9 +228,10 @@ pub(super) mod _serde { use serde::{Deserialize, Serialize}; use super::{Operation, Snapshot, Summary}; + use crate::Error; + use crate::error::invalid_data; use crate::spec::SchemaId; use crate::spec::snapshot::SnapshotRowRange; - use crate::{Error, ErrorKind}; #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] @@ -381,15 +376,13 @@ pub(super) mod _serde { manifest_list: match (v1.manifest_list, v1.manifests) { (Some(file), None) => file, (Some(_), Some(_)) => { - return Err(Error::new( - ErrorKind::DataInvalid, - "Invalid v1 snapshot, when manifest list provided, manifest files should be omitted", + return Err(invalid_data!( + "Invalid v1 snapshot, when manifest list provided, manifest files should be omitted" )); } (None, _) => { - return Err(Error::new( - ErrorKind::DataInvalid, - "Unsupported v1 snapshot, only manifest list is supported", + return Err(invalid_data!( + "Unsupported v1 snapshot, only manifest list is supported" )); } }, diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index c9fd022724..a48a5abdda 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -20,6 +20,7 @@ use std::collections::HashMap; use itertools::Itertools; use super::{DataContentType, DataFile, PartitionSpecRef}; +use crate::error::invalid_data; use crate::spec::{ManifestContentType, ManifestFile, Operation, SchemaRef, Summary}; use crate::{Error, ErrorKind, Result}; @@ -339,10 +340,7 @@ pub(crate) fn update_snapshot_summaries( && summary.operation != Operation::Overwrite && summary.operation != Operation::Delete { - return Err(Error::new( - ErrorKind::DataInvalid, - "Operation is not supported.", - )); + return Err(invalid_data!("Operation is not supported.")); } let mut summary = match previous_summary { diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 84bb97a328..24f10355bf 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -39,7 +39,7 @@ use super::{ }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; -use crate::error::{Result, timestamp_ms_to_utc}; +use crate::error::{Result, invalid_data, timestamp_ms_to_utc}; use crate::io::FileIO; use crate::partitioning::compute_unified_partition_type; use crate::spec::EncryptedKey; @@ -487,12 +487,9 @@ impl TableMetadata { let decompressed_data = CompressionCodec::gzip_default() .decompress(metadata_content.to_vec()) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Trying to read compressed metadata file", - ) - .with_context("file_path", metadata_location) - .with_source(e) + invalid_data!("Trying to read compressed metadata file") + .with_context("file_path", metadata_location) + .with_source(e) })?; serde_json::from_slice(&decompressed_data)? } else { @@ -514,13 +511,10 @@ impl TableMetadata { let codec = self.table_properties().metadata_compression_codec()?; if codec != metadata_location.compression_codec() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Compression codec mismatch: metadata_location has {:?}, but table properties specify {:?}", - metadata_location.compression_codec(), - codec - ), + return Err(invalid_data!( + "Compression codec mismatch: metadata_location has {:?}, but table properties specify {:?}", + metadata_location.compression_codec(), + codec )); } @@ -529,9 +523,8 @@ impl TableMetadata { CompressionCodec::Gzip(_) => codec.compress(json_data)?, CompressionCodec::None => json_data, _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Unsupported metadata compression codec: {codec:?}"), + return Err(invalid_data!( + "Unsupported metadata compression codec: {codec:?}" )); } }; @@ -573,14 +566,11 @@ impl TableMetadata { if field.transform != Transform::Void && self.current_schema().field_by_id(field.source_id).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Default partition spec {} references missing source field {} in current schema {}", - self.default_spec.spec_id(), - field.source_id, - self.current_schema_id - ), + return Err(invalid_data!( + "Default partition spec {} references missing source field {} in current schema {}", + self.default_spec.spec_id(), + field.source_id, + self.current_schema_id )); } } @@ -618,12 +608,9 @@ impl TableMetadata { } if self.default_sort_order_id != SortOrder::UNSORTED_ORDER_ID { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "No sort order exists with the default sort order id {}.", - self.default_sort_order_id - ), + return Err(invalid_data!( + "No sort order exists with the default sort order id {}.", + self.default_sort_order_id )); } @@ -636,12 +623,9 @@ impl TableMetadata { /// Validate the current schema is set and exists. fn validate_current_schema(&self) -> Result<()> { if self.schema_by_id(self.current_schema_id).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "No schema exists with the current schema id {}.", - self.current_schema_id - ), + return Err(invalid_data!( + "No schema exists with the current schema id {}.", + self.current_schema_id )); } Ok(()) @@ -653,11 +637,8 @@ impl TableMetadata { if current_snapshot_id == EMPTY_SNAPSHOT_ID { self.current_snapshot_id = None; } else if self.snapshot_by_id(current_snapshot_id).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Snapshot for current snapshot id {current_snapshot_id} does not exist in the existing snapshots list" - ), + return Err(invalid_data!( + "Snapshot for current snapshot id {current_snapshot_id} does not exist in the existing snapshots list" )); } } @@ -668,11 +649,8 @@ impl TableMetadata { fn validate_refs(&self) -> Result<()> { for (name, snapshot_ref) in self.refs.iter() { if self.snapshot_by_id(snapshot_ref.snapshot_id).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Snapshot for reference {name} does not exist in the existing snapshots list" - ), + return Err(invalid_data!( + "Snapshot for reference {name} does not exist in the existing snapshots list" )); } } @@ -682,19 +660,15 @@ impl TableMetadata { if let Some(main_ref) = main_ref && main_ref.snapshot_id != self.current_snapshot_id.unwrap_or_default() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Current snapshot id does not match main branch ({:?} != {:?})", - self.current_snapshot_id.unwrap_or_default(), - main_ref.snapshot_id - ), + return Err(invalid_data!( + "Current snapshot id does not match main branch ({:?} != {:?})", + self.current_snapshot_id.unwrap_or_default(), + main_ref.snapshot_id )); } } else if main_ref.is_some() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Current snapshot is not set, but main branch exists", + return Err(invalid_data!( + "Current snapshot is not set, but main branch exists" )); } @@ -704,12 +678,9 @@ impl TableMetadata { /// Validate that for V1 Metadata the last_sequence_number is 0 fn validate_snapshot_sequence_number(&self) -> Result<()> { if self.format_version < FormatVersion::V2 && self.last_sequence_number != 0 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Last sequence number must be 0 in v1. Found {}", - self.last_sequence_number - ), + return Err(invalid_data!( + "Last sequence number must be 0 in v1. Found {}", + self.last_sequence_number )); } @@ -719,14 +690,11 @@ impl TableMetadata { .values() .find(|snapshot| snapshot.sequence_number() > self.last_sequence_number) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid snapshot with id {} and sequence number {} greater than last sequence number {}", - snapshot.snapshot_id(), - snapshot.sequence_number(), - self.last_sequence_number - ), + return Err(invalid_data!( + "Invalid snapshot with id {} and sequence number {} greater than last sequence number {}", + snapshot.snapshot_id(), + snapshot.sequence_number(), + self.last_sequence_number )); } @@ -740,10 +708,7 @@ impl TableMetadata { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if curr.timestamp_ms - prev.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - "Expected sorted snapshot log entries", - )); + return Err(invalid_data!("Expected sorted snapshot log entries")); } } @@ -751,12 +716,10 @@ impl TableMetadata { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if self.last_updated_ms - last.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid update timestamp {}: before last snapshot log entry at {}", - self.last_updated_ms, last.timestamp_ms - ), + return Err(invalid_data!( + "Invalid update timestamp {}: before last snapshot log entry at {}", + self.last_updated_ms, + last.timestamp_ms )); } } @@ -769,10 +732,7 @@ impl TableMetadata { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if curr.timestamp_ms - prev.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - "Expected sorted metadata log entries", - )); + return Err(invalid_data!("Expected sorted metadata log entries")); } } @@ -780,12 +740,10 @@ impl TableMetadata { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if self.last_updated_ms - last.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid update timestamp {}: before last metadata log entry at {}", - self.last_updated_ms, last.timestamp_ms - ), + return Err(invalid_data!( + "Invalid update timestamp {}: before last metadata log entry at {}", + self.last_updated_ms, + last.timestamp_ms )); } } @@ -821,6 +779,7 @@ pub(super) mod _serde { DEFAULT_PARTITION_SPEC_ID, EMPTY_SNAPSHOT_ID, FormatVersion, MAIN_BRANCH, MetadataLog, SnapshotLog, TableMetadata, }; + use crate::error::invalid_data; use crate::spec::schema::_serde::{SchemaV1, SchemaV2}; use crate::spec::snapshot::_serde::{SnapshotV1, SnapshotV2, SnapshotV3}; use crate::spec::{ @@ -1018,12 +977,9 @@ pub(super) mod _serde { let current_schema: &SchemaRef = schemas.get(&value.current_schema_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "No schema exists with the current schema id {}.", - value.current_schema_id - ), + invalid_data!( + "No schema exists with the current schema id {}.", + value.current_schema_id ) })?; let partition_specs = HashMap::from_iter( @@ -1040,12 +996,7 @@ pub(super) mod _serde { (DEFAULT_PARTITION_SPEC_ID == default_spec_id) .then(PartitionSpec::unpartition_spec) }) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Default partition spec {default_spec_id} not found"), - ) - })? + .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))? .into(); let default_partition_type = default_spec.partition_type(current_schema)?; @@ -1131,12 +1082,9 @@ pub(super) mod _serde { let current_schema: &SchemaRef = schemas.get(&value.current_schema_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "No schema exists with the current schema id {}.", - value.current_schema_id - ), + invalid_data!( + "No schema exists with the current schema id {}.", + value.current_schema_id ) })?; let partition_specs = HashMap::from_iter( @@ -1153,12 +1101,7 @@ pub(super) mod _serde { (DEFAULT_PARTITION_SPEC_ID == default_spec_id) .then(PartitionSpec::unpartition_spec) }) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Default partition spec {default_spec_id} not found"), - ) - })? + .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))? .into(); let default_partition_type = default_spec.partition_type(current_schema)?; @@ -1248,9 +1191,8 @@ pub(super) mod _serde { let schema = schema_map .get(&schema_id) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("No schema exists with the current schema id {schema_id}."), + invalid_data!( + "No schema exists with the current schema id {schema_id}." ) })? .clone(); @@ -1264,9 +1206,8 @@ pub(super) mod _serde { (schema_map, schema_id, schema_arc) } else { // Option 3: No valid schema configuration found - return Err(Error::new( - ErrorKind::DataInvalid, - "No valid schema configuration found in table metadata", + return Err(invalid_data!( + "No valid schema configuration found in table metadata" )); }; @@ -1303,12 +1244,7 @@ pub(super) mod _serde { let default_spec: PartitionSpecRef = partition_specs .get(&default_spec_id) .map(|x| Arc::unwrap_or_clone(x.clone())) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Default partition spec {default_spec_id} not found"), - ) - })? + .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))? .into(); let default_partition_type = default_spec.partition_type(¤t_schema)?; diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 6467014ab8..3ed848d2d6 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -27,7 +27,7 @@ use super::{ StatisticsFile, StructType, TableMetadata, TableProperties, UNPARTITIONED_LAST_ASSIGNED_ID, UnboundPartitionSpec, }; -use crate::error::{Error, ErrorKind, Result}; +use crate::error::{Error, ErrorKind, Result, invalid_data}; use crate::spec::{EncryptedKey, INITIAL_ROW_ID, MIN_FORMAT_VERSION_ROW_LINEAGE}; use crate::{TableCreation, TableUpdate}; @@ -175,12 +175,8 @@ impl TableMetadataBuilder { format_version, } = table_creation; - let location = location.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't create table without location", - ) - })?; + let location = + location.ok_or_else(|| invalid_data!("Can't create table without location"))?; let partition_spec = partition_spec.unwrap_or(UnboundPartitionSpec { spec_id: None, fields: vec![], @@ -212,12 +208,10 @@ impl TableMetadataBuilder { /// - Cannot downgrade to older format versions. pub fn upgrade_format_version(mut self, format_version: FormatVersion) -> Result { if format_version < self.metadata.format_version { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot downgrade FormatVersion from {} to {}", - self.metadata.format_version, format_version - ), + return Err(invalid_data!( + "Cannot downgrade FormatVersion from {} to {}", + self.metadata.format_version, + format_version )); } @@ -261,12 +255,9 @@ impl TableMetadataBuilder { .collect::>(); if !reserved_properties.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Table properties should not contain reserved properties, but got: [{}]", - reserved_properties.join(", ") - ), + return Err(invalid_data!( + "Table properties should not contain reserved properties, but got: [{}]", + reserved_properties.join(", ") )); } @@ -299,12 +290,9 @@ impl TableMetadataBuilder { .collect::>(); if !reserved_properties.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Table properties to remove contain reserved properties: [{}]", - reserved_properties.join(", ") - ), + return Err(invalid_data!( + "Table properties to remove contain reserved properties: [{}]", + reserved_properties.join(", ") )); } @@ -348,9 +336,9 @@ impl TableMetadataBuilder { .snapshots .contains_key(&snapshot.snapshot_id()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Snapshot already exists for: '{}'", snapshot.snapshot_id()), + return Err(invalid_data!( + "Snapshot already exists for: '{}'", + snapshot.snapshot_id() )); } @@ -358,13 +346,10 @@ impl TableMetadataBuilder { && snapshot.sequence_number() <= self.metadata.last_sequence_number && snapshot.parent_snapshot_id().is_some() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add snapshot with sequence number {} older than last sequence number {}", - snapshot.sequence_number(), - self.metadata.last_sequence_number - ), + return Err(invalid_data!( + "Cannot add snapshot with sequence number {} older than last sequence number {}", + snapshot.sequence_number(), + self.metadata.last_sequence_number )); } @@ -372,13 +357,10 @@ impl TableMetadataBuilder { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if snapshot.timestamp_ms() - last.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid snapshot timestamp {}: before last snapshot timestamp {}", - snapshot.timestamp_ms(), - last.timestamp_ms - ), + return Err(invalid_data!( + "Invalid snapshot timestamp {}: before last snapshot timestamp {}", + snapshot.timestamp_ms(), + last.timestamp_ms )); } } @@ -388,13 +370,10 @@ impl TableMetadataBuilder { .unwrap_or_default() .max(self.metadata.last_updated_ms); if snapshot.timestamp_ms() - max_last_updated < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid snapshot timestamp {}: before last updated timestamp {}", - snapshot.timestamp_ms(), - max_last_updated - ), + return Err(invalid_data!( + "Invalid snapshot timestamp {}: before last updated timestamp {}", + snapshot.timestamp_ms(), + max_last_updated )); } @@ -402,22 +381,16 @@ impl TableMetadataBuilder { if self.metadata.format_version >= MIN_FORMAT_VERSION_ROW_LINEAGE { if let Some((first_row_id, added_rows_count)) = snapshot.row_range() { if first_row_id < self.metadata.next_row_id { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add a snapshot, first-row-id is behind table next-row-id: {first_row_id} < {}", - self.metadata.next_row_id - ), + return Err(invalid_data!( + "Cannot add a snapshot, first-row-id is behind table next-row-id: {first_row_id} < {}", + self.metadata.next_row_id )); } added_rows = Some(added_rows_count); } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add a snapshot: first-row-id is null. first-row-id must be set for format version >= {MIN_FORMAT_VERSION_ROW_LINEAGE}", - ), + return Err(invalid_data!( + "Cannot add a snapshot: first-row-id is null. first-row-id must be set for format version >= {MIN_FORMAT_VERSION_ROW_LINEAGE}", )); } } @@ -428,9 +401,8 @@ impl TableMetadataBuilder { .next_row_id .checked_add(added_rows) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot add snapshot: next-row-id overflowed when adding added-rows", + invalid_data!( + "Cannot add snapshot: next-row-id overflowed when adding added-rows" ) })?; } @@ -459,9 +431,8 @@ impl TableMetadataBuilder { let reference = if let Some(mut reference) = reference { if !reference.is_branch() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot append snapshot to non-branch reference '{branch}'",), + return Err(invalid_data!( + "Cannot append snapshot to non-branch reference '{branch}'", )); } @@ -525,12 +496,9 @@ impl TableMetadataBuilder { } let Some(snapshot) = self.metadata.snapshots.get(&reference.snapshot_id) else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot set '{ref_name}' to unknown snapshot: '{}'", - reference.snapshot_id - ), + return Err(invalid_data!( + "Cannot set '{ref_name}' to unknown snapshot: '{}'", + reference.snapshot_id )); }; @@ -687,9 +655,8 @@ impl TableMetadataBuilder { pub fn set_current_schema(mut self, mut schema_id: i32) -> Result { if schema_id == Self::LAST_ADDED { schema_id = self.last_added_schema_id.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot set current schema to last added schema: no schema has been added.", + invalid_data!( + "Cannot set current schema to last added schema: no schema has been added." ) })?; }; @@ -700,10 +667,7 @@ impl TableMetadataBuilder { } let _schema = self.metadata.schemas.get(&schema_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Cannot set current schema to unknown schema with id: '{schema_id}'"), - ) + invalid_data!("Cannot set current schema to unknown schema with id: '{schema_id}'") })?; // Old partition specs and sort-orders should be preserved even if they are not compatible with the new schema, @@ -749,12 +713,9 @@ impl TableMetadataBuilder { let is_new_field = !self.metadata.name_exists_in_any_schema(field_name); if has_partition_conflict && is_new_field { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add schema field '{field_name}' because it conflicts with existing partition field name. \ + return Err(invalid_data!( + "Cannot add schema field '{field_name}' because it conflicts with existing partition field name. \ Schema evolution cannot introduce field names that match existing partition field names." - ), )); } } @@ -794,23 +755,19 @@ impl TableMetadataBuilder { let has_matching_source_id = schema_field.id == partition_field.source_id; if !is_identity_transform { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot create partition with name '{}' that conflicts with schema field and is not an identity transform.", - partition_field.name - ), + return Err(invalid_data!( + "Cannot create partition with name '{}' that conflicts with schema field and is not an identity transform.", + partition_field.name )); } if !has_matching_source_id { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot create identity partition sourced from different field in schema. \ + return Err(invalid_data!( + "Cannot create identity partition sourced from different field in schema. \ Field name '{}' has id `{}` in schema but partition source id is `{}`", - partition_field.name, schema_field.id, partition_field.source_id - ), + partition_field.name, + schema_field.id, + partition_field.source_id )); } } @@ -858,9 +815,8 @@ impl TableMetadataBuilder { } if self.metadata.format_version <= FormatVersion::V1 && !spec.has_sequential_ids() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot add partition spec with non-sequential field ids to format version 1 table", + return Err(invalid_data!( + "Cannot add partition spec with non-sequential field ids to format version 1 table" )); } @@ -926,9 +882,8 @@ impl TableMetadataBuilder { pub fn set_default_partition_spec(mut self, mut spec_id: i32) -> Result { if spec_id == Self::LAST_ADDED { spec_id = self.last_added_spec_id.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot set default partition spec to last added spec: no spec has been added.", + invalid_data!( + "Cannot set default partition spec to last added spec: no spec has been added." ) })?; } @@ -938,9 +893,8 @@ impl TableMetadataBuilder { } if !self.metadata.partition_specs.contains_key(&spec_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot set default partition spec to unknown spec with id: '{spec_id}'",), + return Err(invalid_data!( + "Cannot set default partition spec to unknown spec with id: '{spec_id}'", )); } @@ -949,11 +903,8 @@ impl TableMetadataBuilder { .partition_specs .get(&spec_id) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot set default partition spec to unknown spec with id: '{spec_id}'", - ), + invalid_data!( + "Cannot set default partition spec to unknown spec with id: '{spec_id}'", ) })? .clone(); @@ -987,10 +938,7 @@ impl TableMetadataBuilder { /// - Cannot remove the default partition spec. pub fn remove_partition_specs(mut self, spec_ids: &[i32]) -> Result { if spec_ids.contains(&self.metadata.default_spec.spec_id()) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot remove default partition spec", - )); + return Err(invalid_data!("Cannot remove default partition spec")); } let mut removed_specs = Vec::with_capacity(spec_ids.len()); @@ -1040,11 +988,8 @@ impl TableMetadataBuilder { .with_fields(sort_order.fields) .build(&schema) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Sort order to add is incompatible with current schema: {e}"), - ) - .with_source(e) + invalid_data!("Sort order to add is incompatible with current schema: {e}") + .with_source(e) })?; self.last_added_order_id = Some(new_order_id); @@ -1064,9 +1009,8 @@ impl TableMetadataBuilder { pub fn set_default_sort_order(mut self, mut sort_order_id: i64) -> Result { if sort_order_id == Self::LAST_ADDED as i64 { sort_order_id = self.last_added_order_id.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot set default sort order to last added order: no order has been added.", + invalid_data!( + "Cannot set default sort order to last added order: no order has been added." ) })?; } @@ -1076,11 +1020,8 @@ impl TableMetadataBuilder { } if !self.metadata.sort_orders.contains_key(&sort_order_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot set default sort order to unknown order with id: '{sort_order_id}'" - ), + return Err(invalid_data!( + "Cannot set default sort order to unknown order with id: '{sort_order_id}'" )); } @@ -1217,9 +1158,8 @@ impl TableMetadataBuilder { if let Some(current_snapshot_id) = self.metadata.current_snapshot_id { let last_id = new_snapshot_log.last().map(|entry| entry.snapshot_id); if last_id != Some(current_snapshot_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot set invalid snapshot log: latest entry is not the current snapshot", + return Err(invalid_data!( + "Cannot set invalid snapshot log: latest entry is not the current snapshot" )); } }; @@ -1289,12 +1229,10 @@ impl TableMetadataBuilder { let mut fresh_spec = PartitionSpecBuilder::new(fresh_schema.clone()); for field in spec.fields() { let source_field_name = previous_id_to_name.get(&field.source_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot find source column with id {} for partition column {} in schema.", - field.source_id, field.name - ), + invalid_data!( + "Cannot find source column with id {} for partition column {} in schema.", + field.source_id, + field.name ) })?; fresh_spec = @@ -1306,12 +1244,9 @@ impl TableMetadataBuilder { let mut fresh_order = SortOrder::builder(); for mut field in sort_order.fields { let source_field_name = previous_id_to_name.get(&field.source_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot find source column with id {} for sort column in schema.", - field.source_id - ), + invalid_data!( + "Cannot find source column with id {} for sort column in schema.", + field.source_id ) })?; let new_field_id = fresh_schema @@ -1354,12 +1289,9 @@ impl TableMetadataBuilder { .schemas .get(&self.metadata.current_schema_id) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Current schema with id '{}' not found in table metadata.", - self.metadata.current_schema_id - ), + invalid_data!( + "Current schema with id '{}' not found in table metadata.", + self.metadata.current_schema_id ) }) } @@ -1370,12 +1302,9 @@ impl TableMetadataBuilder { .get(&self.metadata.default_sort_order_id) .cloned() .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Default sort order with id '{}' not found in table metadata.", - self.metadata.default_sort_order_id - ), + invalid_data!( + "Default sort order with id '{}' not found in table metadata.", + self.metadata.default_sort_order_id ) }) } @@ -1424,10 +1353,7 @@ impl TableMetadataBuilder { /// Does nothing if a schema id is not present. Active schemas should not be removed. pub fn remove_schemas(mut self, schema_id_to_remove: &[i32]) -> Result { if schema_id_to_remove.contains(&self.metadata.current_schema_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot remove current schema", - )); + return Err(invalid_data!("Cannot remove current schema")); } if schema_id_to_remove.is_empty() { diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 01a330e28d..09bf71c196 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -21,13 +21,13 @@ use iceberg_property_macro::properties_view; use crate::compression::CompressionCodec; use crate::encryption::AesKeySize; -use crate::error::{Error, ErrorKind, Result}; +use crate::error::{Result, invalid_data}; use crate::spec::NameMapping; use crate::util::location::strip_trailing_slash; fn parse_location_property(path: &str) -> Result { if path.is_empty() { - return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty")); + return Err(invalid_data!("path must not be empty")); } Ok(strip_trailing_slash(path).to_string()) @@ -43,30 +43,22 @@ fn parse_metadata_compression(value: &str) -> Result { let lowercase_value = value.to_lowercase(); // Use serde to parse the codec (which has rename_all = "lowercase") - let codec: CompressionCodec = serde_json::from_value(serde_json::Value::String( - lowercase_value, - )) - .map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( + let codec: CompressionCodec = + serde_json::from_value(serde_json::Value::String(lowercase_value)).map_err(|_| { + invalid_data!( "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", CompressionCodec::None.name(), CompressionCodec::gzip_default().name() - ), - ) - })?; + ) + })?; // Validate that only None and Gzip are used for metadata match codec { CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", - CompressionCodec::None.name(), - CompressionCodec::gzip_default().name() - ), + _ => Err(invalid_data!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", + CompressionCodec::None.name(), + CompressionCodec::gzip_default().name() )), } } @@ -85,12 +77,9 @@ fn parse_parquet_compression( .get(codec_key) .map(|value| { serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid Parquet compression codec: {value}. Supported codecs: \ + invalid_data!( + "Invalid Parquet compression codec: {value}. Supported codecs: \ uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd" - ), ) }) }) @@ -100,12 +89,9 @@ fn parse_parquet_compression( let level = properties .get(level_key) .map(|value| { - value.parse::().map_err(|error| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid value for {level_key}: {error}"), - ) - }) + value + .parse::() + .map_err(|error| invalid_data!("Invalid value for {level_key}: {error}")) }) .transpose()?; @@ -560,6 +546,7 @@ impl TableProperties<'_> { #[cfg(test)] mod tests { use super::*; + use crate::ErrorKind; use crate::compression::CompressionCodec; #[test] diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index af539b32a2..c4c915c080 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -29,7 +29,7 @@ use super::values::decimal_utils::decimal_from_i128_with_scale; use super::values::temporal::date; use super::{Datum, PrimitiveLiteral}; use crate::ErrorKind; -use crate::error::{Error, Result}; +use crate::error::{Error, Result, invalid_data}; use crate::expr::{ BinaryExpression, BoundPredicate, BoundReference, Predicate, PredicateOperator, Reference, SetExpression, UnaryExpression, @@ -282,9 +282,8 @@ impl Transform { if matches!(input_type, Type::Primitive(_)) { Ok(input_type.clone()) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of identity transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of identity transform", )) } } @@ -306,15 +305,13 @@ impl Transform { | PrimitiveType::Uuid | PrimitiveType::Fixed(_) | PrimitiveType::Binary => Ok(Type::Primitive(PrimitiveType::Int)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of bucket transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of bucket transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of bucket transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of bucket transform", )) } } @@ -326,15 +323,13 @@ impl Transform { | PrimitiveType::String | PrimitiveType::Binary | PrimitiveType::Decimal { .. } => Ok(input_type.clone()), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of truncate transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of truncate transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of truncate transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of truncate transform", )) } } @@ -346,15 +341,13 @@ impl Transform { | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs | PrimitiveType::Date => Ok(Type::Primitive(PrimitiveType::Int)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )) } } @@ -366,15 +359,13 @@ impl Transform { | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs | PrimitiveType::Date => Ok(Type::Primitive(PrimitiveType::Date)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )) } } @@ -385,15 +376,13 @@ impl Transform { | PrimitiveType::Timestamptz | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs => Ok(Type::Primitive(PrimitiveType::Int)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )) } } @@ -496,12 +485,9 @@ impl Transform { PrimitiveLiteral::String(s) => s.len(), PrimitiveLiteral::Binary(b) => b.len(), _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Expected a string or binary literal, got: {:?}", - expr.literal() - ), + return Err(invalid_data!( + "Expected a string or binary literal, got: {:?}", + expr.literal() )); } }; @@ -523,12 +509,9 @@ impl Transform { PrimitiveLiteral::String(s) => s.len(), PrimitiveLiteral::Binary(b) => b.len(), _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Expected a string or binary literal, got: {:?}", - expr.literal() - ), + return Err(invalid_data!( + "Expected a string or binary literal, got: {:?}", + expr.literal() )); } }; @@ -1032,9 +1015,8 @@ impl Transform { | &PrimitiveType::TimestampNs | &PrimitiveType::TimestamptzNs ) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Expected a numeric literal, got: {boundary:?}"), + return Err(invalid_data!( + "Expected a numeric literal, got: {boundary:?}" )); } @@ -1139,11 +1121,7 @@ impl FromStr for Transform { .trim_end_matches(']') .parse() .map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - format!("transform bucket type {v:?} is invalid"), - ) - .with_source(err) + invalid_data!("transform bucket type {v:?} is invalid").with_source(err) })?; Transform::Bucket(length) @@ -1156,20 +1134,13 @@ impl FromStr for Transform { .trim_end_matches(']') .parse() .map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - format!("transform truncate type {v:?} is invalid"), - ) - .with_source(err) + invalid_data!("transform truncate type {v:?} is invalid").with_source(err) })?; Transform::Truncate(width) } v => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("transform {v:?} is invalid"), - )); + return Err(invalid_data!("transform {v:?} is invalid")); } }; diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index 2890029065..57d9675736 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -36,10 +36,10 @@ use super::literal::Literal; use super::primitive::PrimitiveLiteral; use super::serde::_serde::RawLiteral; use super::temporal::{date, time, timestamp, timestamptz}; -use crate::error::Result; +use crate::ensure_data_valid; +use crate::error::{Error, ErrorKind, Result, invalid_data}; use crate::spec::MAX_DECIMAL_PRECISION; use crate::spec::datatypes::{PrimitiveType, Type}; -use crate::{Error, ErrorKind, ensure_data_valid}; /// Maximum value for [`PrimitiveType::Time`] type in microseconds, e.g. 23 hours 59 minutes 59 seconds 999999 microseconds. pub(crate) const MAX_TIME_VALUE: i64 = 24 * 60 * 60 * 1_000_000i64 - 1; @@ -425,14 +425,10 @@ impl Datum { } PrimitiveType::Fixed(_) => PrimitiveLiteral::Binary(Vec::from(bytes)), PrimitiveType::Binary => PrimitiveLiteral::Binary(Vec::from(bytes)), - PrimitiveType::Decimal { .. } => { - PrimitiveLiteral::Int128(i128_from_be_bytes(bytes).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't convert bytes to i128: {bytes:?}"), - ) - })?) - } + PrimitiveType::Decimal { .. } => PrimitiveLiteral::Int128( + i128_from_be_bytes(bytes) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), }; Ok(Datum::new(data_type, literal)) } @@ -458,23 +454,17 @@ impl Datum { PrimitiveLiteral::Binary(val) => ByteBuf::from(val.as_slice()), PrimitiveLiteral::Int128(val) => { let PrimitiveType::Decimal { precision, .. } = self.r#type else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "PrimitiveLiteral Int128 must be PrimitiveType Decimal but got {}", - self.r#type - ), + return Err(invalid_data!( + "PrimitiveLiteral Int128 must be PrimitiveType Decimal but got {}", + self.r#type )); }; // It's required by iceberg spec that we must keep the minimum // number of bytes for the value let Ok(required_bytes) = Type::decimal_required_bytes(precision) else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "PrimitiveType Decimal must has valid precision but got {precision}" - ), + return Err(invalid_data!( + "PrimitiveType Decimal must has valid precision but got {precision}" )); }; @@ -487,9 +477,8 @@ impl Datum { ByteBuf::from(bytes) } PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot convert AboveMax or BelowMin to bytes".to_string(), + return Err(invalid_data!( + "Cannot convert AboveMax or BelowMin to bytes" )); } }; @@ -532,9 +521,10 @@ impl Datum { /// ); /// ``` pub fn bool_from_str>(s: S) -> Result { - let v = s.as_ref().parse::().map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse string to bool.").with_source(e) - })?; + let v = s + .as_ref() + .parse::() + .map_err(|e| invalid_data!("Can't parse string to bool.").with_source(e))?; Ok(Self::bool(v)) } @@ -646,11 +636,7 @@ impl Datum { /// ``` pub fn date_from_str>(s: S) -> Result { let t = s.as_ref().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse date from string: {}", s.as_ref()), - ) - .with_source(e) + invalid_data!("Can't parse date from string: {}", s.as_ref()).with_source(e) })?; Ok(Self::date(date::date_from_naive_date(t))) @@ -671,10 +657,7 @@ impl Datum { /// ``` pub fn date_from_ymd(year: i32, month: u32, day: u32) -> Result { let t = NaiveDate::from_ymd_opt(year, month, day).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't create date from year: {year}, month: {month}, day: {day}"), - ) + invalid_data!("Can't create date from year: {year}, month: {month}, day: {day}") })?; Ok(Self::date(date::date_from_naive_date(t))) @@ -744,11 +727,7 @@ impl Datum { /// ``` pub fn time_from_str>(s: S) -> Result { let t = s.as_ref().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse time from string: {}", s.as_ref()), - ) - .with_source(e) + invalid_data!("Can't parse time from string: {}", s.as_ref()).with_source(e) })?; Ok(Self::time_from_naive_time(t)) @@ -767,10 +746,7 @@ impl Datum { /// ``` pub fn time_from_hms_micro(hour: u32, min: u32, sec: u32, micro: u32) -> Result { let t = NaiveTime::from_hms_micro_opt(hour, min, sec, micro) - .ok_or_else(|| Error::new( - ErrorKind::DataInvalid, - format!("Can't create time from hour: {hour}, min: {min}, second: {sec}, microsecond: {micro}"), - ))?; + .ok_or_else(|| invalid_data!("Can't create time from hour: {hour}, min: {min}, second: {sec}, microsecond: {micro}"))?; Ok(Self::time_from_naive_time(t)) } @@ -842,9 +818,10 @@ impl Datum { /// assert_eq!(&format!("{t}"), "1992-03-01 01:02:03.000088"); /// ``` pub fn timestamp_from_str>(s: S) -> Result { - let dt = s.as_ref().parse::().map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse timestamp.").with_source(e) - })?; + let dt = s + .as_ref() + .parse::() + .map_err(|e| invalid_data!("Can't parse timestamp.").with_source(e))?; Ok(Self::timestamp_from_datetime(dt)) } @@ -911,9 +888,8 @@ impl Datum { /// assert_eq!(&format!("{t}"), "1992-02-29 17:02:03.000088 UTC"); /// ``` pub fn timestamptz_from_str>(s: S) -> Result { - let dt = DateTime::::from_str(s.as_ref()).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse datetime.").with_source(e) - })?; + let dt = DateTime::::from_str(s.as_ref()) + .map_err(|e| invalid_data!("Can't parse datetime.").with_source(e))?; Ok(Self::timestamptz_from_datetime(dt)) } @@ -965,11 +941,7 @@ impl Datum { /// ``` pub fn uuid_from_str>(s: S) -> Result { let uuid = uuid::Uuid::parse_str(s.as_ref()).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse uuid from string: {}", s.as_ref()), - ) - .with_source(e) + invalid_data!("Can't parse uuid from string: {}", s.as_ref()).with_source(e) })?; Ok(Self::uuid(uuid)) } @@ -1103,18 +1075,17 @@ impl Datum { } fn string_to_i128>(s: S) -> Result { - s.as_ref().parse::().map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse string to i128.").with_source(e) - }) + s.as_ref() + .parse::() + .map_err(|e| invalid_data!("Can't parse string to i128.").with_source(e)) } fn decimal_from_mantissa(mantissa: i128, precision: u32, scale: u32) -> Result { let r#type = Type::decimal(precision, scale)?; if decimal_precision(mantissa) > precision { let value = decimal_from_i128_with_scale(mantissa, scale); - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Decimal value {value} is too large for precision {precision}"), + return Err(invalid_data!( + "Decimal value {value} is too large for precision {precision}" )); } @@ -1180,11 +1151,8 @@ impl Datum { scale: target_scale, .. }, - ) => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Decimal scale conversion is not supported: source scale {self_scale}, target scale {target_scale}" - ), + ) => Err(invalid_data!( + "Decimal scale conversion is not supported: source scale {self_scale}, target scale {target_scale}" )), (PrimitiveLiteral::String(val), _, PrimitiveType::Boolean) => { Datum::bool_from_str(val) @@ -1204,21 +1172,17 @@ impl Datum { // TODO: implement more type conversions (_, self_type, target_type) if self_type == target_type => Ok(self), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Can't convert datum from {} type to {} type.", - self.r#type, target_primitive_type - ), + _ => Err(invalid_data!( + "Can't convert datum from {} type to {} type.", + self.r#type, + target_primitive_type )), } } - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Can't convert datum from {} type to {} type.", - self.r#type, target_type - ), + _ => Err(invalid_data!( + "Can't convert datum from {} type to {} type.", + self.r#type, + target_type )), } } diff --git a/crates/iceberg/src/spec/values/decimal_utils.rs b/crates/iceberg/src/spec/values/decimal_utils.rs index 3d09d6f25e..8bb90c973d 100644 --- a/crates/iceberg/src/spec/values/decimal_utils.rs +++ b/crates/iceberg/src/spec/values/decimal_utils.rs @@ -23,7 +23,8 @@ use fastnum::D128; use fastnum::decimal::Context; -use crate::{Error, ErrorKind, Result}; +use crate::Result; +use crate::error::invalid_data; /// Re-export D128 as the Decimal type for use throughout the crate. pub type Decimal = D128; @@ -92,8 +93,7 @@ pub fn decimal_new(mantissa: i64, scale: u32) -> Decimal { /// /// This is equivalent to rust_decimal's `Decimal::from_str_exact`. pub fn decimal_from_str_exact(s: &str) -> Result { - D128::from_str(s, Context::default()) - .map_err(|e| Error::new(ErrorKind::DataInvalid, format!("Can't parse decimal: {e}"))) + D128::from_str(s, Context::default()).map_err(|e| invalid_data!("Can't parse decimal: {e}")) } /// Get the mantissa (unscaled coefficient) as i128. diff --git a/crates/iceberg/src/spec/values/literal.rs b/crates/iceberg/src/spec/values/literal.rs index 63d6c1aee5..1d27ae9953 100644 --- a/crates/iceberg/src/spec/values/literal.rs +++ b/crates/iceberg/src/spec/values/literal.rs @@ -32,9 +32,8 @@ use super::decimal_utils::{ use super::primitive::PrimitiveLiteral; use super::struct_value::Struct; use super::temporal::{date, time, timestamp, timestamptz}; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::datatypes::{PrimitiveType, Type}; -use crate::{Error, ErrorKind}; /// Values present in iceberg type #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -80,9 +79,10 @@ impl Literal { /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Boolean(false)), t); /// ``` pub fn bool_from_str>(s: S) -> Result { - let v = s.as_ref().parse::().map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse string to bool.").with_source(e) - })?; + let v = s + .as_ref() + .parse::() + .map_err(|e| invalid_data!("Can't parse string to bool.").with_source(e))?; Ok(Self::Primitive(PrimitiveLiteral::Boolean(v))) } @@ -164,11 +164,7 @@ impl Literal { /// ``` pub fn date_from_str>(s: S) -> Result { let t = s.as_ref().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse date from string: {}", s.as_ref()), - ) - .with_source(e) + invalid_data!("Can't parse date from string: {}", s.as_ref()).with_source(e) })?; Ok(Self::date(date::date_from_naive_date(t))) @@ -188,10 +184,7 @@ impl Literal { /// ``` pub fn date_from_ymd(year: i32, month: u32, day: u32) -> Result { let t = NaiveDate::from_ymd_opt(year, month, day).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't create date from year: {year}, month: {month}, day: {day}"), - ) + invalid_data!("Can't create date from year: {year}, month: {month}, day: {day}") })?; Ok(Self::date(date::date_from_naive_date(t))) @@ -230,11 +223,7 @@ impl Literal { /// ``` pub fn time_from_str>(s: S) -> Result { let t = s.as_ref().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse time from string: {}", s.as_ref()), - ) - .with_source(e) + invalid_data!("Can't parse time from string: {}", s.as_ref()).with_source(e) })?; Ok(Self::time_from_naive_time(t)) @@ -253,10 +242,7 @@ impl Literal { /// ``` pub fn time_from_hms_micro(hour: u32, min: u32, sec: u32, micro: u32) -> Result { let t = NaiveTime::from_hms_micro_opt(hour, min, sec, micro) - .ok_or_else(|| Error::new( - ErrorKind::DataInvalid, - format!("Can't create time from hour: {hour}, min: {min}, second: {sec}, microsecond: {micro}"), - ))?; + .ok_or_else(|| invalid_data!("Can't create time from hour: {hour}, min: {min}, second: {sec}, microsecond: {micro}"))?; Ok(Self::time_from_naive_time(t)) } @@ -314,18 +300,16 @@ impl Literal { /// assert_eq!(t, t2); /// ``` pub fn timestamp_from_str>(s: S) -> Result { - let dt = DateTime::::from_str(s.as_ref()).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse datetime.").with_source(e) - })?; + let dt = DateTime::::from_str(s.as_ref()) + .map_err(|e| invalid_data!("Can't parse datetime.").with_source(e))?; Ok(Self::timestamp_from_datetime(dt)) } /// Similar to [`Literal::timestamp_from_str`], but return timestamp with timezone literal. pub fn timestamptz_from_str>(s: S) -> Result { - let dt = DateTime::::from_str(s.as_ref()).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse datetime.").with_source(e) - })?; + let dt = DateTime::::from_str(s.as_ref()) + .map_err(|e| invalid_data!("Can't parse datetime.").with_source(e))?; Ok(Self::timestamptz_from_datetime(dt)) } @@ -354,11 +338,7 @@ impl Literal { /// ``` pub fn uuid_from_str>(s: S) -> Result { let uuid = Uuid::parse_str(s.as_ref()).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse uuid from string: {}", s.as_ref()), - ) - .with_source(e) + invalid_data!("Can't parse uuid from string: {}", s.as_ref()).with_source(e) })?; Ok(Self::uuid(uuid)) } @@ -435,30 +415,31 @@ impl Literal { Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( number .as_i64() - .ok_or(Error::new( - ErrorKind::DataInvalid, - "Failed to convert json number to int", - ))? + .ok_or(invalid_data!("Failed to convert json number to int"))? .try_into()?, )))) } - (PrimitiveType::Long, JsonValue::Number(number)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::Long(number.as_i64().ok_or(Error::new( - ErrorKind::DataInvalid, - "Failed to convert json number to long", - ))?), - ))), + (PrimitiveType::Long, JsonValue::Number(number)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::Long( + number + .as_i64() + .ok_or(invalid_data!("Failed to convert json number to long"))?, + )))) + } (PrimitiveType::Float, JsonValue::Number(number)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::Float(OrderedFloat(number.as_f64().ok_or(Error::new( - ErrorKind::DataInvalid, - "Failed to convert json number to float", - ))? as f32)), + PrimitiveLiteral::Float(OrderedFloat( + number + .as_f64() + .ok_or(invalid_data!("Failed to convert json number to float"))? + as f32, + )), ))), (PrimitiveType::Double, JsonValue::Number(number)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::Double(OrderedFloat(number.as_f64().ok_or(Error::new( - ErrorKind::DataInvalid, - "Failed to convert json number to double", - ))?)), + PrimitiveLiteral::Double(OrderedFloat( + number + .as_f64() + .ok_or(invalid_data!("Failed to convert json number to double"))?, + )), ))), (PrimitiveType::Date, JsonValue::String(s)) => { Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( @@ -469,9 +450,8 @@ impl Literal { Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( number .as_i64() - .ok_or(Error::new( - ErrorKind::DataInvalid, - "Failed to convert json number to date (days since epoch)", + .ok_or(invalid_data!( + "Failed to convert json number to date (days since epoch)" ))? .try_into()?, )))) @@ -496,11 +476,8 @@ impl Literal { (PrimitiveType::TimestampNs, JsonValue::String(s)) => { let ndt = NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f")?; let nanos = timestamp::datetime_to_nanoseconds(&ndt).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Timestamp is outside the representable nanosecond range: {ndt}" - ), + invalid_data!( + "Timestamp is outside the representable nanosecond range: {ndt}" ) })?; Ok(Some(Literal::Primitive(PrimitiveLiteral::Long(nanos)))) @@ -511,11 +488,8 @@ impl Literal { "%Y-%m-%dT%H:%M:%S%.f+00:00", )?); let nanos = timestamptz::datetimetz_to_nanoseconds(&dt).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Timestamptz is outside the representable nanosecond range: {dt}" - ), + invalid_data!( + "Timestamptz is outside the representable nanosecond range: {dt}" ) })?; Ok(Some(Literal::Primitive(PrimitiveLiteral::Long(nanos)))) @@ -547,14 +521,12 @@ impl Literal { decimal_mantissa(&rescaled), )))) } - (PrimitiveType::Unknown, value) if !value.is_null() => Err(Error::new( - ErrorKind::DataInvalid, - "Unknown type only supports null default values", + (PrimitiveType::Unknown, value) if !value.is_null() => Err(invalid_data!( + "Unknown type only supports null default values" )), (_, JsonValue::Null) => Ok(None), - (i, j) => Err(Error::new( - ErrorKind::DataInvalid, - format!("The json value {j} doesn't fit to the iceberg type {i}."), + (i, j) => Err(invalid_data!( + "The json value {j} doesn't fit to the iceberg type {i}." )), }, Type::Struct(schema) => { @@ -569,9 +541,8 @@ impl Literal { .collect::>>()?; Ok(Some(Literal::Struct(Struct::from_iter(values)))) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The json value for a struct type must be an object.", + Err(invalid_data!( + "The json value for a struct type must be an object." )) } } @@ -586,9 +557,8 @@ impl Literal { .collect::>>()?, ))) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The json value for a list type must be an array.", + Err(invalid_data!( + "The json value for a list type must be an array." )) } } @@ -598,13 +568,10 @@ impl Literal { (object.remove("keys"), object.remove("values")) { if keys.len() != values.len() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Map keys and values must have the same length, got {} keys and {} values", - keys.len(), - values.len() - ), + return Err(invalid_data!( + "Map keys and values must have the same length, got {} keys and {} values", + keys.len(), + values.len() )); } @@ -615,9 +582,8 @@ impl Literal { Ok(( Literal::try_from_json(key, &map.key_field.field_type) .and_then(|value| { - value.ok_or(Error::new( - ErrorKind::DataInvalid, - "Key of map cannot be null", + value.ok_or(invalid_data!( + "Key of map cannot be null" )) })?, Literal::try_from_json(value, &map.value_field.field_type)?, @@ -626,21 +592,18 @@ impl Literal { .collect::>>()?, )))) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The json value for a list type must be an array.", + Err(invalid_data!( + "The json value for a list type must be an array." )) } } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The json value for a list type must be an array.", + Err(invalid_data!( + "The json value for a list type must be an array." )) } } - Type::Variant(_) => Err(Error::new( - ErrorKind::DataInvalid, - "Variant type is not supported for single-value JSON serialization", + Type::Variant(_) => Err(invalid_data!( + "Variant type is not supported for single-value JSON serialization" )), } } @@ -721,14 +684,12 @@ impl Literal { let decimal = try_decimal_from_i128_with_scale(val, *scale)?; Ok(JsonValue::String(decimal.to_string())) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "The iceberg type for decimal literal must be decimal.", + _ => Err(invalid_data!( + "The iceberg type for decimal literal must be decimal." ))?, }, - _ => Err(Error::new( - ErrorKind::DataInvalid, - "The iceberg value doesn't fit to the iceberg type.", + _ => Err(invalid_data!( + "The iceberg value doesn't fit to the iceberg type." )), }, (Literal::Struct(s), Type::Struct(struct_type)) => { @@ -765,9 +726,8 @@ impl Literal { object.insert("values".to_string(), JsonValue::Array(json_values)); Ok(JsonValue::Object(object)) } - (value, r#type) => Err(Error::new( - ErrorKind::DataInvalid, - format!("The iceberg value {value:?} doesn't fit to the iceberg type {type}."), + (value, r#type) => Err(invalid_data!( + "The iceberg value {value:?} doesn't fit to the iceberg type {type}." )), } } @@ -794,9 +754,8 @@ impl Literal { fn decode_hex_bytes(value: &str) -> Result> { if !value.len().is_multiple_of(2) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Hex string must have an even number of characters: {value:?}"), + return Err(invalid_data!( + "Hex string must have an even number of characters: {value:?}" )); } @@ -816,9 +775,8 @@ fn decode_hex_digit(digit: u8, value: &str) -> Result { b'0'..=b'9' => Ok(digit - b'0'), b'a'..=b'f' => Ok(digit - b'a' + 10), b'A'..=b'F' => Ok(digit - b'A' + 10), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Hex string contains invalid character: {value:?}"), + _ => Err(invalid_data!( + "Hex string contains invalid character: {value:?}" )), } } @@ -838,9 +796,8 @@ fn validate_fixed_size(actual: usize, expected: u64) -> Result<()> { if actual as u64 == expected { Ok(()) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("Fixed type must be exactly {expected} bytes, got {actual}"), + Err(invalid_data!( + "Fixed type must be exactly {expected} bytes, got {actual}" )) } } diff --git a/crates/iceberg/src/spec/values/serde.rs b/crates/iceberg/src/spec/values/serde.rs index 8a00a79dee..896235b1e3 100644 --- a/crates/iceberg/src/spec/values/serde.rs +++ b/crates/iceberg/src/spec/values/serde.rs @@ -24,9 +24,10 @@ pub(crate) mod _serde { use serde_bytes::ByteBuf; use serde_derive::{Deserialize as DeserializeDerive, Serialize as SerializeDerive}; + use crate::Error; + use crate::error::invalid_data; use crate::spec::values::{Literal, Map, PrimitiveLiteral, Struct}; use crate::spec::{MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, PrimitiveType, Type}; - use crate::{Error, ErrorKind}; #[derive(SerializeDerive, DeserializeDerive, Debug)] #[serde(transparent)] @@ -245,10 +246,7 @@ pub(crate) mod _serde { RawLiteralEnum::Bytes(ByteBuf::from(v.to_be_bytes())) } PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => { - return Err(Error::new( - ErrorKind::DataInvalid, - "Can't convert AboveMax or BelowMax", - )); + return Err(invalid_data!("Can't convert AboveMax or BelowMax")); } }, Literal::Struct(r#struct) => { @@ -263,9 +261,8 @@ pub(crate) mod _serde { RawLiteralEnum::try_from(value, &field.field_type)?, )); } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Can't convert null to required field", + return Err(invalid_data!( + "Can't convert null to required field" )); } } else if let Some(value) = value { @@ -278,10 +275,7 @@ pub(crate) mod _serde { } } } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Type {ty} should be a struct"), - )); + return Err(invalid_data!("Type {ty} should be a struct")); } RawLiteralEnum::Record(Record { required, optional }) } @@ -301,10 +295,7 @@ pub(crate) mod _serde { required: list_ty.element_field.required, }) } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Type {ty} should be a list"), - )); + return Err(invalid_data!("Type {ty} should be a list")); } } Literal::Map(map) => { @@ -325,9 +316,8 @@ pub(crate) mod _serde { .transpose()?, )); } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "literal type is inconsistent with type", + return Err(invalid_data!( + "literal type is inconsistent with type" )); } } @@ -348,7 +338,7 @@ pub(crate) mod _serde { Ok(Some(RawLiteralEnum::Record(Record { required: vec![ (MAP_KEY_FIELD_NAME.to_string(), raw_k), - (MAP_VALUE_FIELD_NAME.to_string(), raw_v.ok_or_else(||Error::new(ErrorKind::DataInvalid, "Map value is required, value cannot be null"))?), + (MAP_VALUE_FIELD_NAME.to_string(), raw_v.ok_or_else(||invalid_data!("Map value is required, value cannot be null"))?), ], optional: vec![], }))) @@ -369,10 +359,7 @@ pub(crate) mod _serde { }) } } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Type {ty} should be a map"), - )); + return Err(invalid_data!("Type {ty} should be a map")); } } }; @@ -381,19 +368,13 @@ pub(crate) mod _serde { pub fn try_into(self, ty: &Type) -> Result, Error> { let invalid_err = |v: &str| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Unable to convert raw literal ({v}) fail convert to type {ty} for: type mismatch" - ), + invalid_data!( + "Unable to convert raw literal ({v}) fail convert to type {ty} for: type mismatch" ) }; let invalid_err_with_reason = |v: &str, reason: &str| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Unable to convert raw literal ({v}) fail convert to type {ty} for: {reason}" - ), + invalid_data!( + "Unable to convert raw literal ({v}) fail convert to type {ty} for: {reason}" ) }; match self { diff --git a/crates/iceberg/src/spec/view_metadata.rs b/crates/iceberg/src/spec/view_metadata.rs index cb691e208c..4a816140c3 100644 --- a/crates/iceberg/src/spec/view_metadata.rs +++ b/crates/iceberg/src/spec/view_metadata.rs @@ -32,8 +32,7 @@ use uuid::Uuid; pub use super::view_metadata_builder::ViewMetadataBuilder; use super::view_version::{ViewVersionId, ViewVersionRef}; use super::{SchemaId, SchemaRef}; -use crate::error::{Result, timestamp_ms_to_utc}; -use crate::{Error, ErrorKind}; +use crate::error::{Result, invalid_data, timestamp_ms_to_utc}; /// Reference to [`ViewMetadata`]. pub type ViewMetadataRef = Arc; @@ -170,12 +169,9 @@ impl ViewMetadata { fn validate_current_version_id(&self) -> Result<()> { if !self.versions.contains_key(&self.current_version_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "No version exists with the current version id {}.", - self.current_version_id - ), + return Err(invalid_data!( + "No version exists with the current version id {}.", + self.current_version_id )); } Ok(()) @@ -184,9 +180,8 @@ impl ViewMetadata { fn validate_current_schema_id(&self) -> Result<()> { let schema_id = self.current_version().schema_id(); if !self.schemas.contains_key(&schema_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("No schema exists with the schema id {schema_id}."), + return Err(invalid_data!( + "No schema exists with the schema id {schema_id}." )); } Ok(()) diff --git a/crates/iceberg/src/spec/view_metadata_builder.rs b/crates/iceberg/src/spec/view_metadata_builder.rs index 38041ca625..a59822e70c 100644 --- a/crates/iceberg/src/spec/view_metadata_builder.rs +++ b/crates/iceberg/src/spec/view_metadata_builder.rs @@ -31,7 +31,7 @@ use super::{ }; use crate::ViewCreation; use crate::catalog::ViewUpdate; -use crate::error::{Error, ErrorKind, Result}; +use crate::error::{Result, invalid_data}; use crate::io::is_truthy; /// Manipulating view metadata. @@ -142,12 +142,10 @@ impl ViewMetadataBuilder { /// - Cannot downgrade to older format versions. pub fn upgrade_format_version(self, format_version: ViewFormatVersion) -> Result { if format_version < self.metadata.format_version { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot downgrade ViewFormatVersion from {} to {}", - self.metadata.format_version, format_version - ), + return Err(invalid_data!( + "Cannot downgrade ViewFormatVersion from {} to {}", + self.metadata.format_version, + format_version )); } @@ -183,9 +181,8 @@ impl ViewMetadataBuilder { pub fn set_current_version_id(mut self, mut version_id: i32) -> Result { if version_id == Self::LAST_ADDED { let Some(last_added_id) = self.last_added_version_id else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot set current version id to last added version: no version has been added.", + return Err(invalid_data!( + "Cannot set current version id to last added version: no version has been added." )); }; version_id = last_added_id; @@ -198,10 +195,7 @@ impl ViewMetadataBuilder { } let version = self.metadata.versions.get(&version_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Cannot set current version to unknown version with id: {version_id}"), - ) + invalid_data!("Cannot set current version to unknown version with id: {version_id}") })?; self.metadata.current_version_id = version_id; @@ -276,10 +270,7 @@ impl ViewMetadataBuilder { let view_version = if view_version.schema_id() == Self::LAST_ADDED { let last_added_schema_id = self.last_added_schema_id.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot set last added schema: no schema has been added", - ) + invalid_data!("Cannot set last added schema: no schema has been added") })?; view_version.with_schema_id(last_added_schema_id) } else { @@ -291,12 +282,9 @@ impl ViewMetadataBuilder { .schemas .contains_key(&view_version.schema_id()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add version with unknown schema: {}", - view_version.schema_id() - ), + return Err(invalid_data!( + "Cannot add version with unknown schema: {}", + view_version.schema_id() )); } @@ -308,13 +296,10 @@ impl ViewMetadataBuilder { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if view_version.timestamp_ms() - last.timestamp_ms() < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid snapshot timestamp {}: before last snapshot timestamp {}", - view_version.timestamp_ms(), - last.timestamp_ms() - ), + return Err(invalid_data!( + "Invalid snapshot timestamp {}: before last snapshot timestamp {}", + view_version.timestamp_ms(), + last.timestamp_ms() )); } } @@ -427,11 +412,8 @@ impl ViewMetadataBuilder { .and_then(|v| v.parse::().ok()) .unwrap_or(1); if num_versions_to_keep < 0 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "{VIEW_PROPERTY_VERSION_HISTORY_SIZE} must be positive but was {num_versions_to_keep}" - ), + return Err(invalid_data!( + "{VIEW_PROPERTY_VERSION_HISTORY_SIZE} must be positive but was {num_versions_to_keep}" )); } @@ -587,14 +569,11 @@ fn require_no_dialect_dropped(previous: &ViewVersion, current: &ViewVersion) -> let updated_dialects = lowercase_sql_dialects_for(current); if !updated_dialects.is_superset(&base_dialects) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot replace view due to loss of view dialects: \nPrevious dialects: {:?}\nNew dialects: {:?}\nSet {} to true to allow dropping dialects.", - Vec::from_iter(base_dialects), - Vec::from_iter(updated_dialects), - VIEW_PROPERTY_REPLACE_DROP_DIALECT_ALLOWED - ), + return Err(invalid_data!( + "Cannot replace view due to loss of view dialects: \nPrevious dialects: {:?}\nNew dialects: {:?}\nSet {} to true to allow dropping dialects.", + Vec::from_iter(base_dialects), + Vec::from_iter(updated_dialects), + VIEW_PROPERTY_REPLACE_DROP_DIALECT_ALLOWED )); } @@ -617,12 +596,9 @@ pub(super) fn require_unique_dialects(view_version: &ViewVersion) -> Result<()> match repr { ViewRepresentation::Sql(sql_repr) => { if !seen_dialects.insert(sql_repr.dialect.to_lowercase()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid view version: Cannot add multiple queries for dialect {}", - sql_repr.dialect - ), + return Err(invalid_data!( + "Invalid view version: Cannot add multiple queries for dialect {}", + sql_repr.dialect )); } } diff --git a/crates/iceberg/src/spec/view_version.rs b/crates/iceberg/src/spec/view_version.rs index a02d58c4ef..af95f74e66 100644 --- a/crates/iceberg/src/spec/view_version.rs +++ b/crates/iceberg/src/spec/view_version.rs @@ -29,9 +29,8 @@ use typed_builder::TypedBuilder; use super::INITIAL_VIEW_VERSION_ID; use super::view_metadata::ViewVersionLog; use crate::catalog::NamespaceIdent; -use crate::error::{Result, timestamp_ms_to_utc}; +use crate::error::{Result, invalid_data, timestamp_ms_to_utc}; use crate::spec::{SchemaId, SchemaRef, ViewMetadata}; -use crate::{Error, ErrorKind}; /// Reference to [`ViewVersion`]. pub type ViewVersionRef = Arc; @@ -116,12 +115,7 @@ impl ViewVersion { pub fn schema(&self, view_metadata: &ViewMetadata) -> Result { view_metadata .schema_by_id(self.schema_id()) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Schema with id {} not found", self.schema_id()), - ) - }) + .ok_or_else(|| invalid_data!("Schema with id {} not found", self.schema_id())) .cloned() } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 21cd8681a7..78c2dff05b 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -22,6 +22,7 @@ use std::sync::Arc; use crate::arrow::ArrowReaderBuilder; use crate::encryption::EncryptionManager; use crate::encryption::kms::KeyManagementClient; +use crate::error::invalid_data; use crate::inspect::MetadataTable; use crate::io::FileIO; use crate::io::object_cache::ObjectCache; @@ -30,7 +31,7 @@ use crate::scan::TableScanBuilder; use crate::spec::{ ManifestListReader, ManifestReader, SchemaRef, SnapshotRef, TableMetadata, TableMetadataRef, }; -use crate::{Error, ErrorKind, Result, TableIdent}; +use crate::{Result, TableIdent}; /// Builder to create table scan. pub struct TableBuilder { @@ -135,30 +136,26 @@ impl TableBuilder { } = self; let Some(file_io) = file_io else { - return Err(Error::new( - ErrorKind::DataInvalid, - "FileIO must be provided with TableBuilder.file_io()", + return Err(invalid_data!( + "FileIO must be provided with TableBuilder.file_io()" )); }; let Some(metadata) = metadata else { - return Err(Error::new( - ErrorKind::DataInvalid, - "TableMetadataRef must be provided with TableBuilder.metadata()", + return Err(invalid_data!( + "TableMetadataRef must be provided with TableBuilder.metadata()" )); }; let Some(identifier) = identifier else { - return Err(Error::new( - ErrorKind::DataInvalid, - "TableIdent must be provided with TableBuilder.identifier()", + return Err(invalid_data!( + "TableIdent must be provided with TableBuilder.identifier()" )); }; let Some(runtime) = runtime else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Runtime must be provided with TableBuilder.runtime()", + return Err(invalid_data!( + "Runtime must be provided with TableBuilder.runtime()" )); }; @@ -248,12 +245,9 @@ impl Table { /// Returns current metadata location in a result. pub fn metadata_location_result(&self) -> Result<&str> { - self.metadata_location.as_deref().ok_or(Error::new( - ErrorKind::DataInvalid, - format!( - "Metadata location does not exist for table: {}", - self.identifier - ), + self.metadata_location.as_deref().ok_or(invalid_data!( + "Metadata location does not exist for table: {}", + self.identifier )) } @@ -415,6 +409,7 @@ mod tests { use std::fs; use super::*; + use crate::ErrorKind; use crate::encryption::SensitiveBytes; use crate::encryption::kms::MemoryKeyManagementClient; use crate::spec::TableProperties; diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index 0a390d9829..0e08c6b248 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -21,12 +21,13 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::Utc; +use crate::error::invalid_data; use crate::spec::{ MAIN_BRANCH, SnapshotReference, SnapshotRetention, TableMetadata, TableProperties, }; use crate::table::Table; use crate::transaction::action::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind, Result, TableRequirement, TableUpdate}; +use crate::{Error, Result, TableRequirement, TableUpdate}; /// A transaction action that removes snapshots from table metadata. /// @@ -100,9 +101,8 @@ impl ExpireSnapshotsAction { fn plan(&self, table: &Table, properties: &TableProperties<'_>) -> Result { // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. if self.retain_last == Some(0) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Number of snapshots to retain must be at least 1", + return Err(invalid_data!( + "Number of snapshots to retain must be at least 1" )); } @@ -277,7 +277,7 @@ impl ExpireSnapshotsAction { fn reference_error(metadata: &TableMetadata, snapshot_id: i64) -> Error { if metadata.current_snapshot_id() == Some(snapshot_id) { - return Error::new(ErrorKind::DataInvalid, "Cannot expire the current snapshot"); + return invalid_data!("Cannot expire the current snapshot"); } let ref_names: Vec<&str> = metadata .refs @@ -285,10 +285,7 @@ impl ExpireSnapshotsAction { .filter(|(_, snapshot_ref)| snapshot_ref.snapshot_id == snapshot_id) .map(|(ref_name, _)| ref_name.as_str()) .collect(); - Error::new( - ErrorKind::DataInvalid, - format!("Cannot expire snapshot {snapshot_id}: still referenced by {ref_names:?}"), - ) + invalid_data!("Cannot expire snapshot {snapshot_id}: still referenced by {ref_names:?}") } } @@ -306,9 +303,8 @@ impl TransactionAction for ExpireSnapshotsAction { // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). if !properties.gc_enabled()? { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot expire snapshots: gc.enabled is false", + return Err(invalid_data!( + "Cannot expire snapshots: gc.enabled is false" )); } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 1f09a79a08..43e8b25d5e 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -23,7 +23,7 @@ use futures::TryStreamExt; use futures::stream::FuturesUnordered; use uuid::Uuid; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, Snapshot, @@ -139,16 +139,14 @@ impl<'a> SnapshotProducer<'a> { pub(crate) fn validate_added_data_files(&self) -> Result<()> { for data_file in &self.added_data_files { if data_file.content_type() != crate::spec::DataContentType::Data { - return Err(Error::new( - ErrorKind::DataInvalid, - "Only data content type is allowed for fast append", + return Err(invalid_data!( + "Only data content type is allowed for fast append" )); } // Check if the data file partition spec id matches the table default partition spec id. if self.table.metadata().default_partition_spec_id() != data_file.partition_spec_id { - return Err(Error::new( - ErrorKind::DataInvalid, - "Data file partition spec id does not match table default partition spec id", + return Err(invalid_data!( + "Data file partition spec id does not match table default partition spec id" )); } Self::validate_partition_value( @@ -200,12 +198,9 @@ impl<'a> SnapshotProducer<'a> { .await?; if !referenced_files.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add files that are already referenced by table, files: {}", - referenced_files.join(", ") - ), + return Err(invalid_data!( + "Cannot add files that are already referenced by table, files: {}", + referenced_files.join(", ") )); } @@ -281,9 +276,8 @@ impl<'a> SnapshotProducer<'a> { partition_type: &StructType, ) -> Result<()> { if partition_value.fields().len() != partition_type.fields().len() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Partition value is not compatible with partition type", + return Err(invalid_data!( + "Partition value is not compatible with partition type" )); } @@ -297,9 +291,8 @@ impl<'a> SnapshotProducer<'a> { if let Some(value) = value && !field.compatible(&value.as_primitive_literal().unwrap()) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Partition value is not compatible partition type", + return Err(invalid_data!( + "Partition value is not compatible partition type" )); } } diff --git a/crates/iceberg/src/transaction/sort_order.rs b/crates/iceberg/src/transaction/sort_order.rs index 17655fe235..fb4af49276 100644 --- a/crates/iceberg/src/transaction/sort_order.rs +++ b/crates/iceberg/src/transaction/sort_order.rs @@ -19,11 +19,11 @@ use std::sync::Arc; use async_trait::async_trait; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::{NullOrder, SchemaRef, SortDirection, SortField, SortOrder, Transform}; use crate::table::Table; use crate::transaction::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind, TableRequirement, TableUpdate}; +use crate::{TableRequirement, TableUpdate}; /// Represents a sort field whose construction and validation are deferred until commit time. /// This avoids the need to pass a `Table` reference into methods like `asc` or `desc` when @@ -38,20 +38,14 @@ struct PendingSortField { impl PendingSortField { fn to_sort_field(&self, schema: &SchemaRef) -> Result { - let field_id = schema.field_id_by_name(self.name.as_str()).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Cannot find field {} in table schema", self.name), - ) - })?; + let field_id = schema + .field_id_by_name(self.name.as_str()) + .ok_or_else(|| invalid_data!("Cannot find field {} in table schema", self.name))?; // This action intentionally refuses to author all-null (void) sort keys; // this policy does not restrict reading or preserving existing metadata. if matches!(self.transform, Transform::Unknown | Transform::Void) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot sort by transform {}", self.transform), - )); + return Err(invalid_data!("Cannot sort by transform {}", self.transform)); } Ok(SortField::builder() @@ -95,8 +89,9 @@ impl ReplaceSortOrderAction { /// Whether the transform is valid for the column's type is checked at commit time, /// once the table schema is available (mirroring Java's `SortOrder.Builder.build()`). /// `Transform::Unknown` and `Transform::Void` are rejected at commit time with - /// [`ErrorKind::DataInvalid`]. Rejecting `Void` is an intentional authoring policy - /// for this action: an all-null sort key adds no ordering information. + /// [`ErrorKind::DataInvalid`](crate::ErrorKind::DataInvalid). Rejecting `Void` is an + /// intentional authoring policy for this action: an all-null sort key adds no + /// ordering information. /// /// Note: `Term` is currently a plain column reference. Once it becomes /// transform-carrying (#2665), sort-order declaration is expected to converge on diff --git a/crates/iceberg/src/transaction/update_location.rs b/crates/iceberg/src/transaction/update_location.rs index 4842bcfdf9..7d0e2f9545 100644 --- a/crates/iceberg/src/transaction/update_location.rs +++ b/crates/iceberg/src/transaction/update_location.rs @@ -19,9 +19,10 @@ use std::sync::Arc; use async_trait::async_trait; +use crate::error::invalid_data; use crate::table::Table; use crate::transaction::action::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind, Result, TableUpdate}; +use crate::{Result, TableUpdate}; /// A transaction action that sets or updates the location of a table. /// @@ -60,9 +61,8 @@ impl TransactionAction for UpdateLocationAction { if let Some(location) = self.location.clone() { updates = vec![TableUpdate::SetLocation { location }]; } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Location is not set for UpdateLocationAction!", + return Err(invalid_data!( + "Location is not set for UpdateLocationAction!" )); } diff --git a/crates/iceberg/src/transaction/upgrade_format_version.rs b/crates/iceberg/src/transaction/upgrade_format_version.rs index bc0f799012..b833b57220 100644 --- a/crates/iceberg/src/transaction/upgrade_format_version.rs +++ b/crates/iceberg/src/transaction/upgrade_format_version.rs @@ -19,11 +19,12 @@ use std::sync::Arc; use async_trait::async_trait; +use crate::Result; use crate::TableUpdate::UpgradeFormatVersion; +use crate::error::invalid_data; use crate::spec::FormatVersion; use crate::table::Table; use crate::transaction::action::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind, Result}; /// A transaction action to upgrade a table's format version. /// @@ -61,10 +62,7 @@ impl UpgradeFormatVersionAction { impl TransactionAction for UpgradeFormatVersionAction { async fn commit(self: Arc, _table: &Table) -> Result { let format_version = self.format_version.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "FormatVersion is not set for UpgradeFormatVersionAction!", - ) + invalid_data!("FormatVersion is not set for UpgradeFormatVersionAction!") })?; Ok(ActionCommit::new( diff --git a/crates/iceberg/src/transform/temporal.rs b/crates/iceberg/src/transform/temporal.rs index e9831b5423..04a8256f53 100644 --- a/crates/iceberg/src/transform/temporal.rs +++ b/crates/iceberg/src/transform/temporal.rs @@ -27,6 +27,7 @@ use arrow_schema::{DataType, TimeUnit}; use chrono::{DateTime, Datelike, Duration}; use super::TransformFunction; +use crate::error::invalid_data; use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType}; use crate::{Error, ErrorKind, Result}; @@ -49,12 +50,7 @@ impl Year { #[inline] fn timestamp_to_year_micros(timestamp: i64) -> Result { Ok(DateTime::from_timestamp_micros(timestamp) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Fail to convert timestamp to date in year transform", - ) - })? + .ok_or_else(|| invalid_data!("Fail to convert timestamp to date in year transform"))? .year() - UNIX_EPOCH_YEAR) } @@ -120,12 +116,8 @@ impl Month { // unix epoch date: 1970-01-01 // if date > unix epoch date, delta month = (aa - 1) + 12 * (aaaa-1970) // if date < unix epoch date, delta month = (12 - (aa - 1)) + 12 * (1970-aaaa-1) - let date = DateTime::from_timestamp_micros(timestamp).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Fail to convert timestamp to date in month transform", - ) - })?; + let date = DateTime::from_timestamp_micros(timestamp) + .ok_or_else(|| invalid_data!("Fail to convert timestamp to date in month transform"))?; let unix_epoch_date = DateTime::from_timestamp_micros(0) .expect("0 timestamp from unix epoch should be valid"); if date > unix_epoch_date { @@ -228,10 +220,7 @@ impl Day { }; let delta = Duration::new(secs, nanos).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to create 'TimeDelta' from seconds {secs} and nanos {nanos}"), - ) + invalid_data!("Failed to create 'TimeDelta' from seconds {secs} and nanos {nanos}") })?; let days = (delta.num_days() - offset) as i32; @@ -254,10 +243,7 @@ impl Day { }; let delta = Duration::new(secs, nanos).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to create 'TimeDelta' from seconds {secs} and nanos {nanos}"), - ) + invalid_data!("Failed to create 'TimeDelta' from seconds {secs} and nanos {nanos}") })?; let days = (delta.num_days() - offset) as i32; diff --git a/crates/iceberg/src/transform/truncate.rs b/crates/iceberg/src/transform/truncate.rs index f60f8e4939..0f95854ee1 100644 --- a/crates/iceberg/src/transform/truncate.rs +++ b/crates/iceberg/src/transform/truncate.rs @@ -22,6 +22,7 @@ use arrow_schema::DataType; use super::TransformFunction; use crate::Error; +use crate::error::invalid_data; use crate::spec::decimal_utils::decimal_from_i128_with_scale; use crate::spec::{Datum, PrimitiveLiteral}; @@ -69,10 +70,7 @@ impl TransformFunction for Truncate { match input.data_type() { DataType::Int32 => { let width: i32 = self.width.try_into().map_err(|_| { - Error::new( - crate::ErrorKind::DataInvalid, - "width is failed to convert to i32 when truncate Int32Array", - ) + invalid_data!("width is failed to convert to i32 when truncate Int32Array") })?; let res: arrow_array::Int32Array = input .as_any() @@ -151,10 +149,7 @@ impl TransformFunction for Truncate { match input.literal() { PrimitiveLiteral::Int(v) => Ok(Some({ let width: i32 = self.width.try_into().map_err(|_| { - Error::new( - crate::ErrorKind::DataInvalid, - "width is failed to convert to i32 when truncate Int32Array", - ) + invalid_data!("width is failed to convert to i32 when truncate Int32Array") })?; Datum::int(Self::truncate_i32(*v, width)) })), diff --git a/crates/iceberg/src/writer/base_writer/data_file_writer.rs b/crates/iceberg/src/writer/base_writer/data_file_writer.rs index 02dcda4164..c7a988374e 100644 --- a/crates/iceberg/src/writer/base_writer/data_file_writer.rs +++ b/crates/iceberg/src/writer/base_writer/data_file_writer.rs @@ -19,6 +19,7 @@ use arrow_array::RecordBatch; +use crate::error::invalid_data; use crate::spec::{DataContentType, DataFile, PartitionKey}; use crate::writer::file_writer::FileWriterBuilder; use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; @@ -98,12 +99,8 @@ where res.partition(pk.data().clone()); res.partition_spec_id(pk.spec().spec_id()); } - res.build().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to build data file: {e}"), - ) - }) + res.build() + .map_err(|e| invalid_data!("Failed to build data file: {e}")) }) .collect() } else { diff --git a/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs b/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs index e961d46784..84c10b176a 100644 --- a/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs +++ b/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs @@ -27,6 +27,7 @@ use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use crate::arrow::record_batch_projector::RecordBatchProjector; use crate::arrow::schema_to_arrow_schema; +use crate::error::invalid_data; use crate::spec::{DataFile, PartitionKey, Schema, SchemaRef}; use crate::writer::file_writer::FileWriterBuilder; use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; @@ -78,28 +79,21 @@ pub struct EqualityDeleteWriterConfig { /// here produces clearer errors and avoids generating meaningless delete files. fn validate_equality_ids(equality_ids: &[i32], original_schema: &Schema) -> Result<()> { if equality_ids.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Equality delete field ids must not be empty.", + return Err(invalid_data!( + "Equality delete field ids must not be empty." )); } let mut seen = HashSet::with_capacity(equality_ids.len()); for id in equality_ids { if !seen.insert(*id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Duplicate equality delete field id: {id}"), - ) - .with_context("field_id", id.to_string())); + return Err(invalid_data!("Duplicate equality delete field id: {id}") + .with_context("field_id", id.to_string())); } if original_schema.field_by_id(*id).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid equality delete field id: {id}"), - ) - .with_context("field_id", id.to_string())); + return Err(invalid_data!("Invalid equality delete field id: {id}") + .with_context("field_id", id.to_string())); } } @@ -218,12 +212,8 @@ where res.partition(pk.data().clone()); res.partition_spec_id(pk.spec().spec_id()); } - res.build().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to build data file: {e}"), - ) - }) + res.build() + .map_err(|e| invalid_data!("Failed to build data file: {e}")) }) .collect() } else { diff --git a/crates/iceberg/src/writer/base_writer/position_delete_writer.rs b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs index ebe3e95882..29b65b74ad 100644 --- a/crates/iceberg/src/writer/base_writer/position_delete_writer.rs +++ b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs @@ -38,6 +38,7 @@ use arrow_schema::{DataType, Field}; use once_cell::sync::Lazy; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use crate::error::invalid_data; use crate::metadata_columns::{ RESERVED_FIELD_ID_DELETE_FILE_PATH, RESERVED_FIELD_ID_DELETE_FILE_POS, delete_file_path_field, delete_file_pos_field, @@ -95,22 +96,16 @@ fn field_id(field: &Field) -> Result { .metadata() .get(PARQUET_FIELD_ID_META_KEY) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Position delete column `{}` is missing its Iceberg field id metadata.", - field.name() - ), + invalid_data!( + "Position delete column `{}` is missing its Iceberg field id metadata.", + field.name() ) })? .parse::() .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Position delete column `{}` has an invalid field id: {e}", - field.name() - ), + invalid_data!( + "Position delete column `{}` has an invalid field id: {e}", + field.name() ) }) } @@ -121,68 +116,51 @@ fn field_id(field: &Field) -> Result { fn validate_position_delete_batch(batch: &RecordBatch) -> Result<()> { let fields = batch.schema_ref().fields(); if fields.len() != 2 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "This writer supports only the two required position delete columns (`file_path`, `pos`); \ - batches with a different column count (e.g. including the optional `row` column) are not supported. Got {} columns.", - fields.len() - ), + return Err(invalid_data!( + "This writer supports only the two required position delete columns (`file_path`, `pos`); \ + batches with a different column count (e.g. including the optional `row` column) are not supported. Got {} columns.", + fields.len() )); } let path = &fields[0]; let path_id = field_id(path)?; if path_id != RESERVED_FIELD_ID_DELETE_FILE_PATH { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "The first position delete column must be `file_path` (field id {RESERVED_FIELD_ID_DELETE_FILE_PATH}), but got field id {path_id}." - ), + return Err(invalid_data!( + "The first position delete column must be `file_path` (field id {RESERVED_FIELD_ID_DELETE_FILE_PATH}), but got field id {path_id}." )); } // The canonical schema maps Iceberg `string` to `Utf8` and the file writer is // configured with it, so a `LargeUtf8` column has to be cast to `Utf8` first. if path.data_type() != &DataType::Utf8 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "The position delete `file_path` column must be Utf8 (cast it first); got {:?}.", - path.data_type() - ), + return Err(invalid_data!( + "The position delete `file_path` column must be Utf8 (cast it first); got {:?}.", + path.data_type() )); } // Required column: a nullable field could write nulls under a required schema. if path.is_nullable() { - return Err(Error::new( - ErrorKind::DataInvalid, - "The position delete `file_path` column must be required (non-nullable).", + return Err(invalid_data!( + "The position delete `file_path` column must be required (non-nullable)." )); } let pos = &fields[1]; let pos_id = field_id(pos)?; if pos_id != RESERVED_FIELD_ID_DELETE_FILE_POS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "The second position delete column must be `pos` (field id {RESERVED_FIELD_ID_DELETE_FILE_POS}), but got field id {pos_id}." - ), + return Err(invalid_data!( + "The second position delete column must be `pos` (field id {RESERVED_FIELD_ID_DELETE_FILE_POS}), but got field id {pos_id}." )); } if pos.data_type() != &DataType::Int64 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "The position delete `pos` column must be Int64, but got {:?}.", - pos.data_type() - ), + return Err(invalid_data!( + "The position delete `pos` column must be Int64, but got {:?}.", + pos.data_type() )); } if pos.is_nullable() { - return Err(Error::new( - ErrorKind::DataInvalid, - "The position delete `pos` column must be required (non-nullable).", + return Err(invalid_data!( + "The position delete `pos` column must be required (non-nullable)." )); } @@ -282,12 +260,8 @@ where res.partition(pk.data().clone()); res.partition_spec_id(pk.spec().spec_id()); } - res.build().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Failed to build position delete file: {e}"), - ) - }) + res.build() + .map_err(|e| invalid_data!("Failed to build position delete file: {e}")) }) .collect() } else { diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index fbf333c7bf..b874cb95a9 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -40,6 +40,7 @@ use crate::arrow::{ }; use crate::compression::CompressionCodec; use crate::encryption::{EncryptionManager, StandardKeyMetadata}; +use crate::error::invalid_data; use crate::io::{FileIO, FileWrite, OutputFile}; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MapType, @@ -155,11 +156,7 @@ fn parquet_compression(codec: CompressionCodec) -> Result { } fn invalid_level_error(codec: &str, source: impl Into) -> Error { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid {codec} compression level"), - ) - .with_source(source) + invalid_data!("Invalid {codec} compression level").with_source(source) } impl FileWriterBuilder for ParquetWriterBuilder { @@ -288,11 +285,8 @@ impl SchemaVisitor for IndexByParquetPathName { let full_name = self.field_names.iter().map(String::as_str).join("."); let field_id = self.field_id; if let Some(existing_field_id) = self.name_to_id.get(full_name.as_str()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}" - ), + return Err(invalid_data!( + "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}" )); } else { self.name_to_id.insert(full_name, field_id); @@ -426,12 +420,10 @@ impl ParquetWriter { let reader = input_file.reader().await?; let mut parquet_reader = ArrowFileReader::new(file_metadata, reader); - let parquet_metadata = parquet_reader.get_metadata(None).await.map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - format!("Error reading Parquet metadata: {err}"), - ) - })?; + let parquet_metadata = parquet_reader + .get_metadata(None) + .await + .map_err(|err| invalid_data!("Error reading Parquet metadata: {err}"))?; let mut builder = ParquetWriter::parquet_to_data_file_builder( table_metadata.current_schema().clone(), parquet_metadata, @@ -547,22 +539,19 @@ impl ParquetWriter { upper_bounds.get(&field.source_id), ) { if !field.transform.preserves_order() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "cannot infer partition value for non linear partition field (needs to preserve order): {} with transform {}", - field.name, field.transform - ), + return Err(invalid_data!( + "cannot infer partition value for non linear partition field (needs to preserve order): {} with transform {}", + field.name, + field.transform )); } if lower != upper { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "multiple partition values for field {}: lower: {:?}, upper: {:?}", - field.name, lower, upper - ), + return Err(invalid_data!( + "multiple partition values for field {}: lower: {:?}, upper: {:?}", + field.name, + lower, + upper )); } From e5a18042b244e61556e8ac7daa33450f3c97cb07 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 22 Sep 2026 20:55:05 +0000 Subject: [PATCH 10/63] RFC: Stateful Transaction Action (#2620) * rfc only * update rfc * make it more readable * update rfc * update rfc * remove default impl for cleanup * box self --- docs/rfcs/0003_stateful_transaction.md | 442 +++++++++++++++++++++++++ 1 file changed, 442 insertions(+) create mode 100644 docs/rfcs/0003_stateful_transaction.md diff --git a/docs/rfcs/0003_stateful_transaction.md b/docs/rfcs/0003_stateful_transaction.md new file mode 100644 index 0000000000..6e8f5bc3ce --- /dev/null +++ b/docs/rfcs/0003_stateful_transaction.md @@ -0,0 +1,442 @@ + + +# RFC: Stateful Transactions and Snapshot Production + +**Status:** Draft +**Target:** Apache Iceberg Rust (`iceberg-rust`) + +## 1. Motivation and Scope + +Replaying transaction actions after a catalog conflict is sufficient for simple +metadata updates, but snapshot-producing actions benefit from retaining work +across attempts. FastAppend and merging operations such as RowDelta need stable +execution identity, reusable metadata processing, and ownership of generated +files that may require cleanup. + +This RFC establishes a stateful action model and a shared foundation for snapshot +production. The transaction owns catalog refresh and replay; each action execution +owns retry-persistent state; snapshot-producing actions use persistent producers +as that state. + +The proposal fixes lifetimes, ownership, the core action interface, and the +responsibilities of snapshot producers. It does not prescribe cache layouts, +conflict-validation predicates, or the complete semantics of each operation. +Those can be implemented incrementally within these boundaries. + +## 2. Transaction and Action State Model + +### 2.1 Three Lifetimes + +| Lifetime | Responsibility | +| --- | --- | +| Transaction | Catalog refresh, retry policy, action ordering, replay, and the terminal result | +| Action execution | Immutable operation intent and exclusively owned retry-persistent state | +| Attempt | Execution against the current transaction-local table and construction of that attempt's result | + +An action's intent does not change during replay. Its state survives attempts of +one logical execution, while values derived from a particular base are local to +an attempt or reused only under an appropriate validity check. + +### 2.2 Stateful Action Interface + +Each action declares an associated state type. The following interface captures +the lifecycle; the heterogeneous storage adapters are implementation details. + +```rust +#[async_trait] +pub(crate) trait TransactionAction: Clone + Send + Sync + 'static { + type State: Send + Sync + 'static; + + /// Fresh state for one logical execution; infallible and table-independent. + fn new_state(&self) -> Self::State; + + /// One attempt against the current transaction-local table. + async fn commit( + &self, + state: &mut Self::State, + table: &Table, + ) -> Result; + + /// Best-effort cleanup after the transaction reaches a terminal result. + /// Consumes the action and its state: the type system guarantees that + /// no further attempt can run for this entry after cleanup. + async fn cleanup( + self: Box, + state: Self::State, + table: &Table, + status: CommitStatus, + ); +} +``` + +During replay the action is borrowed immutably, its execution state mutably, +and the table per attempt. The terminal `cleanup` instead consumes the action +and its state by value, so replay cannot resume after cleanup. Stateless +actions use `State = ()`. State is created when an action is added to a +transaction. Table-dependent initialization happens during execution; once +resolved, logical snapshot identity remains stable across retries. + +`CommitStatus` describes terminal cleanup safety (§6). Neither the action nor its +state needs to control the transaction's catalog retry loop. + +The action interface and its storage machinery remain `pub(crate)`: external +crates cannot define transaction actions, so the erasure and adapter details +are internal rather than public API commitments. + +### 2.3 Action Entries and Cloning + +A transaction stores one entry per action, pairing immutable intent with its +execution state. Conceptually: + +```rust +struct ActionEntry { + action: Box, + state: A::State, +} +``` + +The transaction must preserve that pairing when storing heterogeneous entries. +A typed entry erased behind an adapter and separately erased action/state values +with a checked pairing are both acceptable implementations. + +Entries own their actions exclusively so that terminal cleanup can consume +them. Retry reuses the same entry state. Cloning a transaction creates a new +execution of the action plan: it duplicates the immutable actions (actions are +`Clone`) and creates fresh state via `new_state()`. Mutable retry state and +generated execution identity are never shared between transaction executions, +and transaction cloning is expected to be rare, so duplicating intent is an +acceptable cost. + +## 3. Retry and Replay Workflow + +`Transaction::commit` owns refresh, retry, and replay. Each attempt starts from a +catalog base, executes actions in application order, and submits their combined +updates and requirements as one `TableCommit`. + +```mermaid +flowchart TD + Start["Begin transaction commit"] --> Refresh["Load latest catalog base"] + Refresh --> Init["Initialize attempt-local table and update accumulators"] + Init --> Action["Execute next action with retained state"] + Action --> Apply["Apply ActionCommit locally and accumulate changes"] + Apply --> More{"More actions?"} + More -->|Yes| Action + More -->|No| Submit["Build TableCommit and submit to catalog"] + Submit -->|Success| Committed["Committed"] + Submit -->|Retryable conflict and budget remains| Retry["Retain state and artifacts; back off"] + Retry --> Refresh + Submit -->|Definitive failure; no retry| Failed["Failed"] + Submit -->|Indeterminate result| Unknown["Unknown"] + Action -->|Definitive execution failure| Failed + Apply -->|Definitive local apply failure| Failed + Committed --> Finish["Call cleanup on every entry"] + Failed --> Finish + Unknown --> Finish +``` + +Later actions observe earlier actions' transaction-local results, even though +none of those results has yet been committed to the catalog. After a conflict, +the next attempt rebuilds the local table from the refreshed base and replays +the complete action sequence. It does not continue accumulating changes on the +previous attempt's local result. + +The diagram omits refresh errors and other error branches for clarity. The +transaction/catalog layer determines retryability and whether a failed request +definitively did not commit. An unresolved catalog outcome is `Unknown`; it must +not be treated as a known conflict or definite failure. + +Replaying an action does not require repeating all of its I/O or metadata +processing. The same persistent state is passed into each attempt, allowing the +action to reuse valid work while producing a new result for the current base. + +## 4. Persistent Snapshot Producers + +### 4.1 Producers as Action State + +Snapshot-producing actions use persistent producers as their associated state: + +| Producer | Role | Initial consumer | +| --- | --- | --- | +| `SimpleSnapshotProducer` | Append-shaped snapshot production | `FastAppendAction` | +| `MergingSnapshotProducer` | Snapshot production involving data/delete file additions and removals | `RowDeltaAction` | + +An action expresses operation intent and selects the checks required by its +semantics and configuration. The producer supplies reusable validation and +metadata-processing capabilities, retains work across retries, and constructs +the snapshot result for an attempt. + +The producers are independent concrete types, not an inheritance hierarchy. +`MergingSnapshotProducer` does not contain `SimpleSnapshotProducer`. Each producer +owns the persistent state needed for its responsibilities; the transaction does +not need to know that an action's state is a producer. + +### 4.2 Shared Snapshot Production + +Common mechanics are shared through stateless helpers: snapshot-ID generation, +metadata path generation, summary construction, manifest-list writing, and +snapshot/update/requirement assembly. These helpers operate on explicit inputs +and do not own retry state or select operation policy. + +This replaces the attempt-scoped producer and generic hook-trait pipeline with +concrete producers and shared functions. Identity, path allocation, reuse, and +artifact ownership remain producer responsibilities. Whether common persistent +fields are extracted into a shared struct is an implementation choice. + +### 4.3 From Action Intent to a Snapshot + +Snapshot production has three logical stages. These describe responsibilities, +not a requirement to put all stages into one `apply()` method. + +1. **The action selects and invokes validation.** For example, a RowDelta action + may require referenced files to remain live or reject conflicting changes + since a starting snapshot. The producer provides shared validation capabilities + and reusable history processing; the action determines which checks apply. + The concrete conflict predicates are outside this RFC. + +2. **The producer prepares the resulting manifests.** It obtains the current + transaction-local manifest set, applies removals through manifest filtering, + generates or reuses manifests for additions, and assembles the complete + resulting set. Filtering updates manifest entries to reflect file removals; + it does not read Parquet files to execute row-level deletes. Unaffected + manifests can be retained, and valid transformation results can be reused. + +3. **The producer constructs the attempt's snapshot.** It determines parent, + sequence number, applicable row-ID allocation, and summary from the current + table and resulting changes. It writes an attempt-specific manifest list and + returns an `ActionCommit`. The transaction applies that result locally and + eventually submits the combined `TableCommit`. + +For example, if the current set is `M1, M2`, an action removes a file described +by `M1` and adds file `X`, the producer may produce `M1′, M2, MX`: `M1′` reflects +the removal, `M2` is unchanged, and `MX` describes the addition. Manifest filter +managers and history-processing helpers support this workflow without defining +the public action semantics themselves. + +## 5. Reuse Across Attempts + +> Retries recompute base-dependent snapshot composition, while reusing previously +> completed work whenever all semantic inputs that produced it remain unchanged. + +### 5.1 Retained Work and Attempt-Specific Results + +| Work or state | Behavior across retry/rebase | +| --- | --- | +| Action intent and resolved logical identity | Preserve for the execution | +| Added manifests and per-manifest transformation results | Reuse while all relevant inputs remain unchanged | +| Metadata reads and processed validation history | Reuse under the same semantic validity rule | +| Parent, sequence number, and row-ID allocation | Determine from the current attempt's table | +| Complete resulting manifest set and snapshot summary | Recompute for the current attempt | +| Manifest list, `ActionCommit`, and `TableCommit` | Construct for the current attempt | + +The main reuse opportunities are expensive reads, decoding, and manifest +materialization. Reconstructing the complete manifest set does not require +discarding those results, nor does it prohibit caching the current snapshot's +input manifests under an appropriate identity. + +An immutable file is a useful reuse boundary, but file identity alone is not +necessarily the full semantic input to a computation. Operation intent, relevant +metadata context, and inherited values must also remain compatible. Each attempt +must establish its required validation coverage and current-state checks; cached +history or an earlier successful check cannot substitute for that obligation. + +This RFC identifies reuse opportunities without requiring a particular cache +inventory, key type, or storage layer. Correct recomputation is an acceptable +fallback whenever reuse cannot be established safely. + +One assumption deserves explicit statement because it is format-dependent. +Reusing a generated added manifest across attempts is valid under current +formats because the attempt-sensitive values — committing snapshot ID, data +sequence number, first-row-id — are inherited through manifest-list metadata +rather than embedded in the manifest file, so a rebase changes what the +manifest inherits without invalidating its content. A future format that +embeds base-dependent values directly into manifests would add those values +to the manifest's semantic inputs and shrink this reuse accordingly. That is +a seam inside producers and their writers; it changes what may be reused, not +the ownership, replay, or cleanup architecture. + +### 5.2 Logical Identity Versus Snapshot Representation + +A stable snapshot ID identifies one logical snapshot-producing execution. It +does not imply that an uncommitted snapshot's representation is stable across +attempts. + +In a multi-action transaction, action B can observe a snapshot produced locally +by action A. After rebase, A retains its snapshot ID but may produce a different +parent, sequence number, manifest set, and manifest-list location. B must use +that new representation. + +Committed snapshots have immutable representations. Transaction-local snapshots +must instead be treated as attempt-dependent until committed. Any reuse involving +them must be guarded by the relevant representation or semantic inputs, or the +work must be recomputed. Snapshot ID alone is insufficient for that purpose. + +### 5.3 Multi-Action Replay Example + +Consider A appending file X and B removing a file described by manifest M1. +Assume the relevant transformation inputs remain compatible across the rebase, +and a concurrent commit adds an unrelated manifest M3. + +| Stage | First attempt | After rebase | +| --- | --- | --- | +| Catalog base | `M1, M2` | `M3, M1, M2` | +| A: append X | Write `MX`; assemble `MX, M1, M2` | Reuse `MX`; assemble `MX, M3, M1, M2` | +| B: remove file | Transform `M1 → M1′`; retain `MX, M2` | Reuse valid results for `M1, MX, M2`; evaluate `M3` | +| B: resulting set | `MX, M1′, M2` | `MX, M3, M1′, M2` if M3 is unaffected | +| Snapshot construction | A and B each write a manifest list | A and B each write a new manifest list | + +A and B replay in full, but neither necessarily rewrites its materialized +manifests. B consumes A's current output, not A's previous attempt's complete +manifest set. If relevant inputs change or validation finds a conflict, the +affected work is recomputed or the action fails according to its semantics. + +## 6. Artifact Ownership and Terminal Cleanup + +Producers create metadata before catalog commit succeeds. Each producer must +track its generated artifacts, including those written by owned helper +components. Caller-provided data or delete files do not become producer-owned +merely because the action references them. + +Generated metadata paths are write-once: a new attempt must not overwrite files +from an earlier attempt. Reused artifacts keep their existing paths. Artifacts +are retained during retry and considered for cleanup only when the transaction +reaches a terminal result. + +```rust +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CommitStatus { + /// The catalog confirmed that the commit was applied. + Committed, + /// The transaction definitively did not commit and will not retry. + Failed, + /// The commit request was submitted, but its outcome could not be + /// resolved: the catalog may or may not have applied it. + /// + /// Example: every action executed and validated successfully, but the + /// connection failed while awaiting the catalog's response to + /// `update_table`. + Unknown, +} +``` + +| Status | Meaning | Cleanup | +| --- | --- | --- | +| `Committed` | The transaction committed successfully | Retain owned artifacts needed by the committed result; clean up the rest | +| `Failed` | The transaction definitively did not commit and will not retry | Clean up owned artifacts | +| `Unknown` | The catalog may have committed, but the result is unresolved | Delete nothing | + +The transaction/catalog layer determines this classification. Producers consume +it rather than independently interpreting catalog errors. Once the retry loop +terminates, every entry receives `cleanup`, including entries that wrote +files before a later action failed. + +For `Committed`, the supplied table reflects the accepted metadata and supports +reachability checks. Cleanup must account for all snapshots and references +retained by the committed transaction, not only its final current snapshot; if +reachability cannot be established safely, cleanup retains the artifact. +For `Failed`, no reachability reasoning is required: the transaction +definitively did not commit, so producer-owned artifacts are deletable. +For `Unknown`, the supplied table is available execution context only — it is +not proof that the commit failed or that a potentially committed artifact is +unreachable — and cleanup deletes nothing. + +`Unknown` arises only at the catalog boundary, after local work has succeeded: +the commit request was submitted, but its result could not be verified — for +example, a network failure while awaiting the catalog's response. The catalog +layer should make a best-effort attempt to resolve the true outcome before +reporting `Unknown` (for example through idempotent request handling or by +re-checking table state), so `Unknown` is a last resort, not a routine result. +When it is reported, the transaction fails with an error and must not be +retried: retrying could commit a second time if the original request landed, +and cleaning up could delete metadata that is now live. + +Cleanup is best-effort and must not change the already-determined transaction +result. Failures are reported without turning a successful commit into a failed +one. Process crashes and abandoned transactions may still leave orphans; orphan +cleanup remains the safety net. Resolving unknown catalog outcomes is deferred. + +## 7. Alternatives Considered + +**Attempt-scoped producer borrowing a separate retry-state struct.** An earlier +draft kept the producer attempt-scoped and had it borrow a long-lived +retry-state bundle owned by the action entry. Field-for-field, that bundle +converges on exactly what a persistent producer holds, while adding a borrow +split and a second struct. The persistent producer expresses the same boundary +more simply: its fields are retry-persistent, and values derived from one base +are locals of one attempt or validity-guarded cache entries. + +**Java-style producer inheritance and hook traits.** The previous attempt-scoped +`SnapshotProducer` used hook traits (`SnapshotProduceOperation`, +`ManifestProcess`) to let operations inject behavior into a generic pipeline — +a template-method pattern standing in for Java's class hierarchy. With concrete +sibling producers whose attempt logic owns the pipeline and takes operation +inputs as arguments, the hooks have no remaining role, and no base type or +internal locking is needed. + +**Producer-typed action state.** `TransactionAction::State` could have been +bounded by a snapshot-producer trait. But many actions are stateless or not +snapshot-producing, and no transaction-level code needs to know that a state is +a producer: replay calls `commit`, terminal cleanup calls `cleanup`, both on +the action. A generic `State` subsumes the producer case without vacuous impls. + +## 8. Adoption Plan and Deferred Decisions + +### 8.1 Implementation Plan + +Implement the architecture incrementally along its dependency boundaries: + +1. **Stateful actions and replay.** Introduce the associated state, action-entry + ownership, and fresh-state cloning. Migrate stateless actions with `State = ()` + and verify that retry retains state while rebuilding attempt-local results. + +2. **Persistent simple snapshot production.** Extract stateless helpers, + introduce `SimpleSnapshotProducer`, and migrate FastAppend. Remove the old + attempt-scoped producer and obsolete hook traits once consumers are migrated. + +3. **Merging and artifact lifecycle.** Introduce `MergingSnapshotProducer`, + manifest filtering, and reusable added-manifest production. Integrate terminal + status reporting and cleanup for both producers. + +4. **Validation capabilities.** Add reusable history processing and shared + validation operations. Specify concrete predicates and their semantics in + companion work. + +5. **End-to-end operations.** Wire RowDelta intent and validation controls into + the producer, then extend to other snapshot operations. Verify multi-action + replay, changed transaction-local snapshot representations, valid manifest + reuse, and cleanup under success, definite failure, and unknown outcomes. + +These are implementation milestones, not mandatory one-to-one PR boundaries. +The goal is to establish and exercise the shared architecture through concrete +operations, without requiring every optimization or operation up front. + +### 8.2 Deferred Implementation Decisions + +The following do not need to be fixed to agree on this architecture: + +- Shared state structs such as `CommitIdentity`, and eager/lazy UUID generation. +- Cache representations, keys, bounds, eviction, and placement in lower layers. +- Manifest-filter internals and per-spec bookkeeping. +- Conflict predicates, isolation levels, and operation-specific validation APIs. +- Manifest merging/organization and reuse of those results. +- Whole-attempt reuse, unknown-outcome resolution, REST idempotency integration, + and cross-process retry state. + +Implementations may evolve these details while preserving the lifecycle, +ownership, replay, reuse, and cleanup boundaries defined above. From 830bb886542527f4e8956f03040e7c407b621401 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Tue, 22 Sep 2026 21:36:02 +0000 Subject: [PATCH 11/63] fix(cache): drop format-version from the manifest-list cache key (#3259) * fix(cache): drop format-version from the manifest-list cache key The manifest-list cache key now uses the location alone; format_version is no longer part of it. ManifestList::parse_with_version(bytes, version) is a pure function of its inputs, the bytes are immutable and uniquely identified by their location, and an ObjectCache is bound to a single table. So format_version can never disambiguate two lookups at the same location. The only case where it could differ for one cache is an in-place format upgrade (v1 -> v2 mid-cache-lifetime). Dropping it is more correct there: the cache serves the original correct v1 parse instead of a miss that would reparse the v1 bytes under v2. Adds test_get_manifest_list_ignores_format_version, asserting two lookups at one location under different format versions return the same cached Arc. Follow-up to the review thread on #3235: https://github.com/apache/iceberg-rust/pull/3235#discussion_r4024075908 * PR feedback --- crates/iceberg/src/io/object_cache.rs | 124 +++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 10 deletions(-) diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 01a556b9cb..dfc6c124c9 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -21,8 +21,8 @@ use std::sync::Arc; use crate::encryption::EncryptionManager; use crate::io::FileIO; use crate::spec::{ - FormatVersion, Manifest, ManifestFile, ManifestList, ManifestListReader, ManifestReader, - SnapshotRef, TableMetadataRef, + Manifest, ManifestFile, ManifestList, ManifestListReader, ManifestReader, SnapshotRef, + TableMetadataRef, }; use crate::{Error, ErrorKind, Result}; @@ -36,7 +36,12 @@ pub(crate) enum CachedItem { #[derive(Clone, Debug, Hash, Eq, PartialEq)] pub(crate) enum CachedObjectKey { - ManifestList((String, FormatVersion)), + // Keyed by location alone. A location holds one set of immutable bytes, and + // a cache is bound to one table, so the first access parses those bytes + // under the table's then-current format version and every later access + // shares that entry. The version can still shift under a live cache across + // an in-place upgrade (`Table::with_metadata`), which is why the key omits it. + ManifestList(String), // The manifest-level `first_row_id` is part of the key because the parsed // manifest inherits it onto its entries: the same physical manifest can be // referenced with different offsets across snapshots and branches, so it @@ -156,10 +161,7 @@ impl ObjectCache { .map(Arc::new); } - let key = CachedObjectKey::ManifestList(( - snapshot.manifest_list().to_string(), - table_metadata.format_version, - )); + let key = CachedObjectKey::ManifestList(snapshot.manifest_list().to_string()); let cache_entry = self .cache .entry_by_ref(&key) @@ -225,9 +227,9 @@ mod tests { use crate::TableIdent; use crate::io::{FileIO, OutputFile}; use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Literal, ManifestEntry, - ManifestListWriter, ManifestStatus, ManifestWriterBuilder, Operation, Snapshot, Struct, - Summary, TableMetadata, + DataContentType, DataFileBuilder, DataFileFormat, FormatVersion, Literal, + ManifestContentType, ManifestEntry, ManifestListWriter, ManifestStatus, + ManifestWriterBuilder, Operation, Snapshot, Struct, Summary, TableMetadata, }; use crate::table::Table; use crate::test_utils::test_runtime; @@ -348,6 +350,60 @@ mod tests { .unwrap(); manifest_list_write.close().await.unwrap(); } + + async fn setup_v1_manifest_files(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // Write data files + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v1(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v1( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } } #[tokio::test] @@ -490,6 +546,54 @@ mod tests { ); } + #[tokio::test] + async fn test_get_manifest_list_serves_cached_v1_parse_after_upgrade() { + let mut fixture = TableTestFixture::new(); + fixture.setup_v1_manifest_files().await; + + let current_snapshot = fixture.table.metadata().current_snapshot().unwrap(); + + // The manifest list on disk is v1. The fixture table is v2, so clone its + // metadata as v1 to model the state before an in-place upgrade; the real + // metadata_ref models the state after it. Both drive lookups at the same + // location through one cache. + let metadata_v2 = fixture.table.metadata_ref(); + let mut metadata_v1 = fixture.table.metadata().clone(); + metadata_v1.format_version = FormatVersion::V1; + let metadata_v1: TableMetadataRef = Arc::new(metadata_v1); + assert_ne!( + metadata_v2.format_version, metadata_v1.format_version, + "setup invariant: the two metadata refs must differ in format version" + ); + + let object_cache = ObjectCache::new(fixture.table.file_io().clone(), None); + + // Cold miss under v1 parses the v1 bytes as v1: content defaults to Data and + // the sequence number to 0. + let cached_under_v1 = object_cache + .get_manifest_list(current_snapshot, &metadata_v1) + .await + .unwrap(); + let entry = cached_under_v1.entries().first().unwrap(); + assert_eq!(entry.content, ManifestContentType::Data); + assert_eq!(entry.sequence_number, 0); + + // Warm hit under v2 at the same location returns the cached v1 parse rather + // than reparsing the v1 bytes as v2, so the upgraded caller sees the same + // correct content. + let cached_under_v2 = object_cache + .get_manifest_list(current_snapshot, &metadata_v2) + .await + .unwrap(); + assert!( + Arc::ptr_eq(&cached_under_v1, &cached_under_v2), + "lookups at one location must share a cache entry across format versions" + ); + let entry = cached_under_v2.entries().first().unwrap(); + assert_eq!(entry.content, ManifestContentType::Data); + assert_eq!(entry.sequence_number, 0); + } + #[tokio::test] async fn test_get_manifest_keys_on_first_row_id() { use crate::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Type}; From c9a25b23b7645f3af0e9dfd7e85d8cf646901c55 Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 23 Sep 2026 21:20:18 +0000 Subject: [PATCH 12/63] fix(encryption) [18/N] wire in ags1 file length for tamper proofing (#3236) * feat(encryption) [18N] wire in ags1 file length for tamper proofing * remove debug * expect * fix test * fix opendal * make more clear * Add oversized tests * don't do unreachable * import fix * Assert length in streaming tests * DRAFT try key_metadata_with_length * fix(encryption): address key metadata length review --- CHANGELOG.md | 9 + crates/iceberg/public-api.txt | 16 +- .../src/arrow/caching_delete_file_loader.rs | 7 +- crates/iceberg/src/encryption/io.rs | 225 ++++++++++++++++-- crates/iceberg/src/encryption/key_metadata.rs | 4 +- crates/iceberg/src/encryption/manager.rs | 8 +- crates/iceberg/src/encryption/stream.rs | 158 +++++++++--- crates/iceberg/src/io/file_io.rs | 4 +- crates/iceberg/src/io/storage/local_fs.rs | 18 +- crates/iceberg/src/io/storage/memory.rs | 10 +- crates/iceberg/src/puffin/reader.rs | 2 +- crates/iceberg/src/puffin/writer.rs | 15 +- crates/iceberg/src/spec/manifest/writer.rs | 107 ++++----- .../src/spec/manifest_list/manifest_file.rs | 17 +- .../iceberg/src/spec/manifest_list/writer.rs | 14 +- crates/iceberg/src/transaction/append.rs | 31 ++- crates/iceberg/src/transaction/snapshot.rs | 21 +- .../src/writer/file_writer/parquet_writer.rs | 2 + .../TableMetadataV3ValidEncryption.json | 2 +- crates/storage/opendal/src/lib.rs | 76 +++++- 20 files changed, 568 insertions(+), 178 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e06bafa91..83eeeb31ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Breaking Changes + +* `FileWrite::close`, `EncryptedOutputFile::write`, `ManifestListWriter::close`, and `PuffinWriter::close` now return `Result` instead of `Result<()>`. The returned size is the on-disk size, including encryption overhead. Custom `FileWrite` implementations must return the number of bytes stored. +* `EncryptedOutputFile::key_metadata()` is replaced by `key_metadata_with_saved_file_metadata(&FileMetadata)`. Pass the metadata returned by `write()` or the writer's `close()` to include the stored length before encoding key metadata. +* `EncryptedInputFile::metadata()` is now synchronous and derives the plaintext size from key metadata without a storage stat. Remove `.await` from calls to this method. +* AGS1 readers now require `StandardKeyMetadata::file_length` and reject missing or invalid lengths without falling back to a storage stat. AGS1-encrypted manifests, manifest lists, and Puffin files written by earlier development builds without this field must be rewritten using a build that can still read them before upgrading. This matches the Java client's read contract. + ## [v0.10.1] - 2026-07-28 * chore: Update crossbeam-epoch to 0.9.20, pin CI ruff version to 0.15.22 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2911 diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index f29c5efe07..e2e78073e9 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -273,7 +273,7 @@ pub struct iceberg::encryption::AesGcmFileWrite impl iceberg::encryption::AesGcmFileWrite pub fn iceberg::encryption::AesGcmFileWrite::new(inner: alloc::boxed::Box, cipher: alloc::sync::Arc, aad_prefix: impl core::convert::Into>) -> Self impl iceberg::io::FileWrite for iceberg::encryption::AesGcmFileWrite -pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub fn iceberg::encryption::AesGcmFileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct iceberg::encryption::EncryptedInputFile impl iceberg::encryption::EncryptedInputFile @@ -281,7 +281,7 @@ pub async fn iceberg::encryption::EncryptedInputFile::exists(&self) -> iceberg:: pub fn iceberg::encryption::EncryptedInputFile::into_inner(self) -> iceberg::io::InputFile pub fn iceberg::encryption::EncryptedInputFile::key_metadata(&self) -> &iceberg::encryption::StandardKeyMetadata pub fn iceberg::encryption::EncryptedInputFile::location(&self) -> &str -pub async fn iceberg::encryption::EncryptedInputFile::metadata(&self) -> iceberg::Result +pub fn iceberg::encryption::EncryptedInputFile::metadata(&self) -> iceberg::Result pub fn iceberg::encryption::EncryptedInputFile::new(inner: iceberg::io::InputFile, key_metadata: iceberg::encryption::StandardKeyMetadata) -> Self pub async fn iceberg::encryption::EncryptedInputFile::read(&self) -> iceberg::Result pub async fn iceberg::encryption::EncryptedInputFile::reader(&self) -> iceberg::Result> @@ -291,10 +291,10 @@ pub struct iceberg::encryption::EncryptedOutputFile impl iceberg::encryption::EncryptedOutputFile pub async fn iceberg::encryption::EncryptedOutputFile::delete(&self) -> iceberg::Result<()> pub fn iceberg::encryption::EncryptedOutputFile::into_inner(self) -> iceberg::io::OutputFile -pub fn iceberg::encryption::EncryptedOutputFile::key_metadata(&self) -> &iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::EncryptedOutputFile::key_metadata_with_saved_file_metadata(&self, file_metadata: &iceberg::io::FileMetadata) -> iceberg::encryption::StandardKeyMetadata pub fn iceberg::encryption::EncryptedOutputFile::location(&self) -> &str pub fn iceberg::encryption::EncryptedOutputFile::new(inner: iceberg::io::OutputFile, key_metadata: iceberg::encryption::StandardKeyMetadata) -> Self -pub async fn iceberg::encryption::EncryptedOutputFile::write(&self, bs: bytes::bytes::Bytes) -> iceberg::Result<()> +pub async fn iceberg::encryption::EncryptedOutputFile::write(&self, bs: bytes::bytes::Bytes) -> iceberg::Result pub async fn iceberg::encryption::EncryptedOutputFile::writer(&self) -> iceberg::Result> impl core::fmt::Debug for iceberg::encryption::EncryptedOutputFile pub fn iceberg::encryption::EncryptedOutputFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -1046,10 +1046,10 @@ pub fn iceberg::encryption::AesGcmFileRead::read<'life0, 'async_trait>(&'life0 s impl + core::marker::Send + core::marker::Sync + core::marker::Unpin + 'static> iceberg::io::FileRead for T pub fn T::read<'life0, 'async_trait>(&'life0 self, range: core::ops::range::Range) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub trait iceberg::io::FileWrite: core::marker::Send + core::marker::Unpin + 'static -pub fn iceberg::io::FileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::io::FileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub fn iceberg::io::FileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait impl iceberg::io::FileWrite for iceberg::encryption::AesGcmFileWrite -pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub fn iceberg::encryption::AesGcmFileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub trait iceberg::io::Storage: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize pub fn iceberg::io::Storage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait @@ -1267,7 +1267,7 @@ pub async fn iceberg::puffin::PuffinReader::new_from_encrypted(encrypted_input: pub struct iceberg::puffin::PuffinWriter impl iceberg::puffin::PuffinWriter pub async fn iceberg::puffin::PuffinWriter::add(&mut self, blob: iceberg::puffin::Blob, compression_codec: iceberg::compression::CompressionCodec) -> iceberg::Result<()> -pub async fn iceberg::puffin::PuffinWriter::close(self) -> iceberg::Result<()> +pub async fn iceberg::puffin::PuffinWriter::close(self) -> iceberg::Result pub async fn iceberg::puffin::PuffinWriter::new(output_file: &iceberg::io::OutputFile, properties: std::collections::hash::map::HashMap, compress_footer: bool) -> iceberg::Result pub async fn iceberg::puffin::PuffinWriter::new_from_encrypted(encrypted_output: &iceberg::encryption::EncryptedOutputFile, properties: std::collections::hash::map::HashMap, compress_footer: bool) -> iceberg::Result pub const iceberg::puffin::APACHE_DATASKETCHES_THETA_V1: &str @@ -2138,7 +2138,7 @@ pub async fn iceberg::spec::ManifestListReader::load(&self) -> iceberg::Result) -> iceberg::Result<()> -pub async fn iceberg::spec::ManifestListWriter::close(self) -> iceberg::Result<()> +pub async fn iceberg::spec::ManifestListWriter::close(self) -> iceberg::Result pub fn iceberg::spec::ManifestListWriter::next_row_id(&self) -> core::option::Option pub fn iceberg::spec::ManifestListWriter::v1(writer: alloc::boxed::Box, snapshot_id: i64, parent_snapshot_id: core::option::Option) -> Self pub fn iceberg::spec::ManifestListWriter::v2(writer: alloc::boxed::Box, snapshot_id: i64, parent_snapshot_id: core::option::Option, sequence_number: i64) -> Self diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 0208a678ea..e1cb6de158 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -1621,13 +1621,16 @@ mod tests { let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") .unwrap() .with_aad_prefix(b"test-aad-prefix!"); - let encoded_key_metadata = key_metadata.encode().unwrap(); let blob = encode_dv_blob([2u64, 4]); let plaintext_size = blob.len() as i64; let dv_path = format!("{table_location}/deletes.puffin"); let output = EncryptedOutputFile::new(file_io.new_output(&dv_path).unwrap(), key_metadata); - output.write(Bytes::from(blob)).await.unwrap(); + let file_metadata = output.write(Bytes::from(blob)).await.unwrap(); + let encoded_key_metadata = output + .key_metadata_with_saved_file_metadata(&file_metadata) + .encode() + .unwrap(); // content_offset / content_size_in_bytes are in the plaintext coordinate space, distinct // from the ciphertext's on-disk size (header, nonce, and tag overhead). diff --git a/crates/iceberg/src/encryption/io.rs b/crates/iceberg/src/encryption/io.rs index 3eafbe82f3..17fc1a1f23 100644 --- a/crates/iceberg/src/encryption/io.rs +++ b/crates/iceberg/src/encryption/io.rs @@ -23,8 +23,9 @@ use bytes::Bytes; use super::crypto::AesGcmCipher; use super::key_metadata::StandardKeyMetadata; -use super::stream::{AesGcmFileRead, AesGcmFileWrite}; +use super::stream::{AesGcmFileRead, AesGcmFileWrite, MIN_STREAM_LENGTH}; use crate::Result; +use crate::error::invalid_data; use crate::io::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; /// An AGS1 stream-encrypted input file wrapping a plain [`InputFile`]. @@ -54,12 +55,11 @@ impl EncryptedInputFile { self.inner.exists().await } - /// Fetch and returns metadata of file. + /// Returns file metadata from the declared encrypted length without performing I/O. /// /// The returned size is the **plaintext** size. - pub async fn metadata(&self) -> Result { - let raw_meta = self.inner.metadata().await?; - let plaintext_size = AesGcmFileRead::calculate_plaintext_length(raw_meta.size)?; + pub fn metadata(&self) -> Result { + let plaintext_size = AesGcmFileRead::calculate_plaintext_length(self.encrypted_length()?)?; Ok(FileMetadata { size: plaintext_size, }) @@ -67,21 +67,34 @@ impl EncryptedInputFile { /// Read and returns whole content of file (decrypted plaintext). pub async fn read(&self) -> Result { - let meta = self.metadata().await?; + let meta = self.metadata()?; let reader = self.reader().await?; reader.read(0..meta.size).await } /// Creates a reader that transparently decrypts on each read. pub async fn reader(&self) -> Result> { - let raw_meta = self.inner.metadata().await?; + let encrypted_length = self.encrypted_length()?; let raw_reader = self.inner.reader().await?; let cipher = build_cipher(&self.key_metadata)?; let aad_prefix: Box<[u8]> = self.key_metadata.aad_prefix().unwrap_or_default().into(); - let decrypting = AesGcmFileRead::new(raw_reader, cipher, aad_prefix, raw_meta.size)?; + let decrypting = AesGcmFileRead::new(raw_reader, cipher, aad_prefix, encrypted_length)?; Ok(Box::new(decrypting)) } + // A storage stat would hide truncation; require the original length from key metadata. + fn encrypted_length(&self) -> Result { + let length = self.key_metadata.file_length().ok_or_else(|| { + invalid_data!("AGS1 key metadata is missing the encrypted file length") + })?; + if length < u64::from(MIN_STREAM_LENGTH) { + return Err(invalid_data!( + "Invalid encrypted file length: {length} is less than {MIN_STREAM_LENGTH}" + )); + } + Ok(length) + } + /// Returns a reference to the file's key metadata. pub fn key_metadata(&self) -> &StandardKeyMetadata { &self.key_metadata @@ -118,9 +131,14 @@ impl EncryptedOutputFile { } } - /// Returns a reference to the file's key metadata. - pub fn key_metadata(&self) -> &StandardKeyMetadata { - &self.key_metadata + /// Returns key metadata using the encrypted size returned by [`FileWrite::close`] or [`Self::write`]. + pub fn key_metadata_with_saved_file_metadata( + &self, + file_metadata: &FileMetadata, + ) -> StandardKeyMetadata { + self.key_metadata + .clone() + .with_file_length(file_metadata.size) } /// Absolute path of the file. @@ -138,8 +156,8 @@ impl EncryptedOutputFile { ))) } - /// Write bytes to file (transparently encrypted). - pub async fn write(&self, bs: Bytes) -> Result<()> { + /// Write bytes to the file and return its encrypted size. + pub async fn write(&self, bs: Bytes) -> Result { let mut writer = self.writer().await?; writer.write(bs).await?; writer.close().await @@ -172,6 +190,10 @@ fn build_cipher(metadata: &StandardKeyMetadata) -> Result> { #[cfg(test)] mod tests { use super::*; + use crate::ErrorKind; + use crate::encryption::stream::{ + CIPHER_BLOCK_SIZE, GCM_STREAM_HEADER_LENGTH, PLAIN_BLOCK_SIZE, + }; use crate::io::FileIO; fn key_metadata() -> StandardKeyMetadata { @@ -187,9 +209,12 @@ mod tests { let plaintext = b"Hello from EncryptedInputFile/EncryptedOutputFile!"; let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); - output.write(Bytes::from(plaintext.to_vec())).await.unwrap(); + let file_metadata = output.write(Bytes::from(plaintext.to_vec())).await.unwrap(); - let input = EncryptedInputFile::new(fileio.new_input(path).unwrap(), key_metadata()); + let input = EncryptedInputFile::new( + fileio.new_input(path).unwrap(), + output.key_metadata_with_saved_file_metadata(&file_metadata), + ); let content = input.read().await.unwrap(); assert_eq!(&content[..], plaintext); } @@ -201,7 +226,7 @@ mod tests { let plaintext = b"some bytes to measure"; let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); - output.write(Bytes::from(plaintext.to_vec())).await.unwrap(); + let file_metadata = output.write(Bytes::from(plaintext.to_vec())).await.unwrap(); let raw_size = fileio .new_input(path) @@ -215,8 +240,172 @@ mod tests { "encrypted file should be larger than plaintext (header + nonce + tag)" ); - let input = EncryptedInputFile::new(fileio.new_input(path).unwrap(), key_metadata()); - let meta = input.metadata().await.unwrap(); + // A missing path proves the size comes from the key metadata rather than a stat call. + let input = EncryptedInputFile::new( + fileio.new_input("memory:///does-not-exist").unwrap(), + output.key_metadata_with_saved_file_metadata(&file_metadata), + ); + let meta = input.metadata().unwrap(); assert_eq!(meta.size, plaintext.len() as u64); } + + #[tokio::test] + async fn test_missing_file_length_is_rejected() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/missing_length.bin"; + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + output.write(Bytes::from_static(b"data")).await.unwrap(); + let input = EncryptedInputFile::new(fileio.new_input(path).unwrap(), key_metadata()); + + for err in [ + input.metadata().err().unwrap(), + input.reader().await.err().unwrap(), + input.read().await.unwrap_err(), + ] { + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("missing the encrypted file length") + ); + } + } + + #[tokio::test] + async fn test_invalid_file_length_is_rejected() { + let fileio = FileIO::new_with_memory(); + for length in [ + 0, + u64::from(GCM_STREAM_HEADER_LENGTH), + u64::from(MIN_STREAM_LENGTH - 1), + ] { + let input = EncryptedInputFile::new( + fileio + .new_input("memory:///test/invalid_length.bin") + .unwrap(), + key_metadata().with_file_length(length), + ); + assert_eq!( + input.metadata().err().unwrap().kind(), + ErrorKind::DataInvalid + ); + assert_eq!( + input.reader().await.err().unwrap().kind(), + ErrorKind::DataInvalid + ); + } + } + + #[tokio::test] + async fn test_oversized_file_length_is_rejected() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/oversized_length.bin"; + let plaintext = Bytes::from_static(b"some bytes to measure"); + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + let file_metadata = output.write(plaintext.clone()).await.unwrap(); + + // A declared length is trusted without a stat, so an inflated one is only caught once a + // read runs off the end of the real file. Both a minimal overstatement and one spanning a + // whole extra block must fail rather than silently return short plaintext. + for excess in [1, u64::from(CIPHER_BLOCK_SIZE)] { + let input = EncryptedInputFile::new( + fileio.new_input(path).unwrap(), + key_metadata().with_file_length(file_metadata.size + excess), + ); + + let inflated_size = input.metadata().unwrap().size; + assert!(inflated_size > plaintext.len() as u64); + + assert_eq!( + input.read().await.unwrap_err().kind(), + ErrorKind::DataInvalid + ); + + // Not even the bytes that genuinely are on disk can be read back. + let reader = input.reader().await.unwrap(); + assert_eq!( + reader + .read(0..plaintext.len() as u64) + .await + .unwrap_err() + .kind(), + ErrorKind::DataInvalid + ); + } + } + + #[tokio::test] + async fn test_truncated_file_is_rejected() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/truncated.bin"; + let plaintext = Bytes::from(vec![42; 2 * PLAIN_BLOCK_SIZE as usize + 17]); + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + let file_metadata = output.write(plaintext.clone()).await.unwrap(); + let metadata = key_metadata().with_file_length(file_metadata.size); + let ciphertext = fileio.new_input(path).unwrap().read().await.unwrap(); + let truncated_length = (GCM_STREAM_HEADER_LENGTH + CIPHER_BLOCK_SIZE) as usize; + fileio + .new_output(path) + .unwrap() + .write(ciphertext.slice(..truncated_length)) + .await + .unwrap(); + + let input = EncryptedInputFile::new(fileio.new_input(path).unwrap(), metadata); + assert_eq!(input.metadata().unwrap().size, plaintext.len() as u64); + let reader = input.reader().await.unwrap(); + assert_eq!( + reader.read(0..u64::from(PLAIN_BLOCK_SIZE)).await.unwrap(), + plaintext.slice(..PLAIN_BLOCK_SIZE as usize) + ); + assert_eq!( + input.read().await.unwrap_err().kind(), + ErrorKind::DataInvalid + ); + } + + #[tokio::test] + async fn test_truncated_empty_file_is_rejected() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/truncated_empty.bin"; + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + let file_metadata = output.write(Bytes::new()).await.unwrap(); + let metadata = key_metadata().with_file_length(file_metadata.size); + let ciphertext = fileio.new_input(path).unwrap().read().await.unwrap(); + fileio + .new_output(path) + .unwrap() + .write(ciphertext.slice(..GCM_STREAM_HEADER_LENGTH as usize)) + .await + .unwrap(); + let input = EncryptedInputFile::new(fileio.new_input(path).unwrap(), metadata); + assert_eq!( + input.read().await.unwrap_err().kind(), + ErrorKind::DataInvalid + ); + } + + #[tokio::test] + async fn test_close_returns_encrypted_size() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/streaming.bin"; + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + for plaintext in [ + Bytes::from(vec![42; PLAIN_BLOCK_SIZE as usize + 17]), + Bytes::new(), + ] { + let mut writer = output.writer().await.unwrap(); + for chunk in plaintext.chunks(1024) { + writer.write(Bytes::copy_from_slice(chunk)).await.unwrap(); + } + let metadata = writer.close().await.unwrap(); + let size = fileio + .new_input(path) + .unwrap() + .metadata() + .await + .unwrap() + .size; + assert_eq!(metadata.size, size); + } + } } diff --git a/crates/iceberg/src/encryption/key_metadata.rs b/crates/iceberg/src/encryption/key_metadata.rs index 4f8445a9a2..d56c8dbd3b 100644 --- a/crates/iceberg/src/encryption/key_metadata.rs +++ b/crates/iceberg/src/encryption/key_metadata.rs @@ -75,7 +75,7 @@ impl StandardKeyMetadata { self } - /// Adds a file length. + /// Sets the encrypted file length in bytes, required for AGS1 truncation protection. pub fn with_file_length(mut self, length: u64) -> Self { self.file_length = Some(length); self @@ -91,7 +91,7 @@ impl StandardKeyMetadata { self.aad_prefix.as_deref() } - /// Returns the optional file length. + /// Returns the optional encrypted file length in bytes; AGS1 readers require it to be set. pub fn file_length(&self) -> Option { self.file_length } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index d6f4e4af6b..47ff981603 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -670,12 +670,16 @@ mod tests { let encrypted_output = mgr.encrypt(output); let plaintext = b"Hello, encrypted Iceberg round-trip!"; - let serialized_metadata = encrypted_output.key_metadata().encode().unwrap(); - encrypted_output + let file_metadata = encrypted_output .write(bytes::Bytes::from(plaintext.to_vec())) .await .unwrap(); + let serialized_metadata = encrypted_output + .key_metadata_with_saved_file_metadata(&file_metadata) + .encode() + .unwrap(); + let input = io.new_input(path).unwrap(); let parsed_metadata = StandardKeyMetadata::decode(&serialized_metadata).unwrap(); let decrypted_file = EncryptedInputFile::new(input, parsed_metadata); diff --git a/crates/iceberg/src/encryption/stream.rs b/crates/iceberg/src/encryption/stream.rs index c14905f70e..14c0d38446 100644 --- a/crates/iceberg/src/encryption/stream.rs +++ b/crates/iceberg/src/encryption/stream.rs @@ -50,7 +50,7 @@ use bytes::{Bytes, BytesMut}; use super::AesGcmCipher; use crate::error::invalid_data; -use crate::io::{FileRead, FileWrite}; +use crate::io::{FileMetadata, FileRead, FileWrite}; use crate::{Error, ErrorKind, Result}; /// Default plaintext block size (1 MiB), matching Java's `Ciphers.PLAIN_BLOCK_SIZE`. @@ -72,8 +72,7 @@ pub const GCM_STREAM_MAGIC: [u8; 4] = *b"AGS1"; pub const GCM_STREAM_HEADER_LENGTH: u32 = 8; /// Minimum valid AGS1 stream length (header + one empty block). -#[cfg(test)] -pub const MIN_STREAM_LENGTH: u32 = GCM_STREAM_HEADER_LENGTH + NONCE_LENGTH + GCM_TAG_LENGTH; +pub(crate) const MIN_STREAM_LENGTH: u32 = GCM_STREAM_HEADER_LENGTH + NONCE_LENGTH + GCM_TAG_LENGTH; /// Constructs the per-block AAD for AGS1 stream encryption. /// @@ -147,20 +146,14 @@ impl AesGcmFileRead { aad_prefix: Box<[u8]>, encrypted_file_length: u64, ) -> Result { + if encrypted_file_length < u64::from(MIN_STREAM_LENGTH) { + return Err(invalid_data!( + "Invalid encrypted file length: {encrypted_file_length} is less than {MIN_STREAM_LENGTH}" + )); + } let plain_stream_size = Self::calculate_plaintext_length(encrypted_file_length)?; let stream_length = encrypted_file_length - GCM_STREAM_HEADER_LENGTH as u64; - if stream_length == 0 { - return Ok(Self { - inner, - cipher, - aad_prefix, - plain_stream_size: 0, - num_blocks: 0, - last_cipher_block_size: 0, - }); - } - let num_full_blocks = stream_length / CIPHER_BLOCK_SIZE as u64; let cipher_bytes_in_last_block = (stream_length % CIPHER_BLOCK_SIZE as u64) as u32; let full_blocks_only = cipher_bytes_in_last_block == 0; @@ -280,7 +273,9 @@ impl FileRead for AesGcmFileRead { /// file. GCM authentication is verified per-block, so any tampering is detected /// at the granularity of individual blocks. async fn read(&self, range: Range) -> Result { - if range.start == range.end { + // An empty stream still has one block whose GCM tag must be verified, so an empty read + // cannot short-circuit there: that is what detects a file truncated to just its header. + if range.start == range.end && self.plain_stream_size != 0 { return Ok(Bytes::new()); } @@ -301,12 +296,8 @@ impl FileRead for AesGcmFileRead { )); } - if self.num_blocks == 0 { - return Ok(Bytes::new()); - } - let first_block = range.start / PLAIN_BLOCK_SIZE as u64; - let last_block = (range.end - 1) / PLAIN_BLOCK_SIZE as u64; + let last_block = range.end.saturating_sub(1) / PLAIN_BLOCK_SIZE as u64; // Read all needed encrypted blocks in a single I/O call let encrypted_start = Self::encrypted_block_offset(first_block); @@ -314,6 +305,13 @@ impl FileRead for AesGcmFileRead { Self::encrypted_block_offset(last_block) + self.cipher_block_size(last_block) as u64; let all_encrypted = self.inner.read(encrypted_start..encrypted_end).await?; + if all_encrypted.len() as u64 != encrypted_end - encrypted_start { + return Err(invalid_data!( + "Invalid encrypted read length: expected {} bytes, got {}", + encrypted_end - encrypted_start, + all_encrypted.len() + )); + } // Decrypt each block and extract the requested plaintext range let result_len = (range.end - range.start) as usize; @@ -489,7 +487,7 @@ impl FileWrite for AesGcmFileWrite { Ok(()) } - async fn close(&mut self) -> Result<()> { + async fn close(&mut self) -> Result { if self.closed { return Err(Error::new( ErrorKind::Unexpected, @@ -624,6 +622,69 @@ mod tests { assert!(result.is_empty()); } + #[tokio::test] + async fn test_short_ciphertext_read_is_rejected() { + struct ShortRead; + + #[async_trait::async_trait] + impl FileRead for ShortRead { + async fn read(&self, range: Range) -> Result { + let len = (range.end - range.start).saturating_sub(1) as usize; + Ok(Bytes::from(vec![0; len])) + } + } + + let reader = AesGcmFileRead::new( + Box::new(ShortRead), + Arc::new(make_cipher(b"0123456789abcdef")), + Box::default(), + u64::from(MIN_STREAM_LENGTH) + 10, + ) + .unwrap(); + let err = reader.read(0..10).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("Invalid encrypted read length")); + } + + #[tokio::test] + async fn test_oversized_declared_length_is_rejected() { + // Object stores clamp reads at EOF rather than failing, unlike the in-memory FileIO. + struct ClampingRead(Bytes); + + #[async_trait::async_trait] + impl FileRead for ClampingRead { + async fn read(&self, range: Range) -> Result { + let start = (range.start as usize).min(self.0.len()); + let end = (range.end as usize).min(self.0.len()); + Ok(self.0.slice(start..end)) + } + } + + let key = b"0123456789abcdef"; + let aad_prefix = b"test-aad-prefix!"; + let plaintext = b"some bytes to measure"; + let encrypted = write_through_ags1(plaintext, key, aad_prefix).await; + + // A declared length larger than the real file must be rejected rather than yielding + // truncated plaintext, whether it overstates by one byte or by a whole extra block. + for excess in [1, u64::from(CIPHER_BLOCK_SIZE)] { + let reader = AesGcmFileRead::new( + Box::new(ClampingRead(Bytes::from(encrypted.clone()))), + Arc::new(make_cipher(key)), + aad_prefix.to_vec().into_boxed_slice(), + encrypted.len() as u64 + excess, + ) + .unwrap(); + + let err = reader + .read(0..plaintext.len() as u64) + .await + .expect_err("an inflated declared length must not read back as plaintext"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("Invalid encrypted read length")); + } + } + #[tokio::test] async fn test_small_file_roundtrip() { let key = b"0123456789abcdef"; @@ -940,15 +1001,21 @@ mod tests { assert_eq!(&aad[..], &42u32.to_le_bytes()); } - #[tokio::test] - async fn test_encrypted_file_too_short() { - let result = AesGcmFileRead::new( - memory_reader(vec![0; 4]), - Arc::new(make_cipher(b"0123456789abcdef")), - [].into(), - 4, - ); - assert!(result.is_err()); + #[test] + fn test_encrypted_file_too_short() { + for length in 0..MIN_STREAM_LENGTH { + let result = AesGcmFileRead::new( + memory_reader(vec![0; length as usize]), + Arc::new(make_cipher(b"0123456789abcdef")), + [].into(), + u64::from(length), + ); + let err = result + .err() + .expect("a stream must contain an authenticated block"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("Invalid encrypted file length")); + } } // --- AesGcmFileWrite tests --- @@ -974,8 +1041,14 @@ mod tests { Ok(()) } - async fn close(&mut self) -> Result<()> { - Ok(()) + // A poisoned AesGcmFileWrite must reject close() before delegating, so reaching this is a + // bug. Returning an error rather than panicking surfaces it as a failed assertion on the + // expected "poisoned" message. + async fn close(&mut self) -> Result { + Err(Error::new( + ErrorKind::Unexpected, + "FailingFileWrite::close called unexpectedly", + )) } } @@ -986,8 +1059,10 @@ mod tests { Ok(()) } - async fn close(&mut self) -> Result<()> { - Ok(()) + async fn close(&mut self) -> Result { + Ok(FileMetadata { + size: self.buffer.lock().unwrap().len() as u64, + }) } } @@ -1001,9 +1076,15 @@ mod tests { let mut writer = AesGcmFileWrite::new(inner, cipher, aad_prefix.to_vec()); writer.write(Bytes::from(plaintext.to_vec())).await.unwrap(); - writer.close().await.unwrap(); + let metadata = writer.close().await.unwrap(); - buffer.lock().unwrap().clone() + let encrypted = buffer.lock().unwrap().clone(); + assert_eq!( + metadata.size, + encrypted.len() as u64, + "close() must report the full ciphertext length" + ); + encrypted } #[tokio::test] @@ -1223,10 +1304,9 @@ mod tests { ); // Close should also be rejected - let result = writer.close().await; - assert!(result.is_err()); + let err = writer.close().await.err().expect("close should fail"); assert!( - result.unwrap_err().to_string().contains("poisoned"), + err.to_string().contains("poisoned"), "expected poisoned error on close" ); } diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 87ac84784f..42eec4db56 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -375,10 +375,10 @@ pub trait FileWrite: Send + Unpin + 'static { /// TODO: we can support writing non-contiguous bytes in the future. async fn write(&mut self, bs: Bytes) -> Result<()>; - /// Close file. + /// Close the file and return its stored size, including encryption overhead for encrypted files. /// /// Calling close on closed file will generate an error. - async fn close(&mut self) -> Result<()>; + async fn close(&mut self) -> Result; } /// Output file is used for writing to files.. diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs index d5ef0a4732..cb780e86e7 100644 --- a/crates/iceberg/src/io/storage/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -249,12 +249,16 @@ impl FileRead for LocalFsFileRead { #[derive(Debug)] pub struct LocalFsFileWrite { file: Option, + bytes_written: u64, } impl LocalFsFileWrite { /// Create a new `LocalFsFileWrite` for the given file. pub fn new(file: fs::File) -> Self { - Self { file: Some(file) } + Self { + file: Some(file), + bytes_written: 0, + } } } @@ -272,11 +276,12 @@ impl FileWrite for LocalFsFileWrite { format!("Failed to write to file: {e}"), ) })?; + self.bytes_written += bs.len() as u64; Ok(()) } - async fn close(&mut self) -> Result<()> { + async fn close(&mut self) -> Result { let file = self .file .take() @@ -285,7 +290,9 @@ impl FileWrite for LocalFsFileWrite { file.sync_all() .map_err(|e| Error::new(ErrorKind::Unexpected, format!("Failed to sync file: {e}")))?; - Ok(()) + Ok(FileMetadata { + size: self.bytes_written, + }) } } @@ -464,10 +471,11 @@ mod tests { let mut writer = storage.writer(path_str).await.unwrap(); writer.write(Bytes::from("Hello, ")).await.unwrap(); writer.write(Bytes::from("World!")).await.unwrap(); - writer.close().await.unwrap(); + let metadata = writer.close().await.unwrap(); let content = storage.read(path_str).await.unwrap(); assert_eq!(content, Bytes::from("Hello, World!")); + assert_eq!(metadata.size, content.len() as u64); } #[tokio::test] @@ -494,7 +502,7 @@ mod tests { let path_str = path.to_str().unwrap(); let mut writer = storage.writer(path_str).await.unwrap(); - writer.close().await.unwrap(); + assert_eq!(writer.close().await.unwrap().size, 0); // Write after close should fail let result = writer.write(Bytes::from("test")).await; diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs index b39c741ea7..fefa7973e8 100644 --- a/crates/iceberg/src/io/storage/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -311,7 +311,7 @@ impl FileWrite for MemoryFileWrite { Ok(()) } - async fn close(&mut self) -> Result<()> { + async fn close(&mut self) -> Result { if self.closed { return Err(invalid_data!("File already closed")); } @@ -323,12 +323,13 @@ impl FileWrite for MemoryFileWrite { ) })?; + let size = self.buffer.len() as u64; data.insert( self.path.clone(), Bytes::from(std::mem::take(&mut self.buffer)), ); self.closed = true; - Ok(()) + Ok(FileMetadata { size }) } } @@ -482,10 +483,11 @@ mod tests { let mut writer = storage.writer(path).await.unwrap(); writer.write(Bytes::from("Hello, ")).await.unwrap(); writer.write(Bytes::from("World!")).await.unwrap(); - writer.close().await.unwrap(); + let metadata = writer.close().await.unwrap(); let content = storage.read(path).await.unwrap(); assert_eq!(content, Bytes::from("Hello, World!")); + assert_eq!(metadata.size, content.len() as u64); } #[tokio::test] @@ -508,7 +510,7 @@ mod tests { let path = "memory://test/file.txt"; let mut writer = storage.writer(path).await.unwrap(); - writer.close().await.unwrap(); + assert_eq!(writer.close().await.unwrap().size, 0); // Write after close should fail let result = writer.write(Bytes::from("test")).await; diff --git a/crates/iceberg/src/puffin/reader.rs b/crates/iceberg/src/puffin/reader.rs index 01601f48e0..a7b178ac84 100644 --- a/crates/iceberg/src/puffin/reader.rs +++ b/crates/iceberg/src/puffin/reader.rs @@ -45,7 +45,7 @@ impl PuffinReader { /// reader operates over plaintext offsets and length, so all blob and /// footer positions match those written to the unencrypted file. pub async fn new_from_encrypted(encrypted_input: EncryptedInputFile) -> Result { - let file_length = encrypted_input.metadata().await?.size; + let file_length = encrypted_input.metadata()?.size; let file_read = encrypted_input.reader().await?; Ok(Self::from_parts(file_read, file_length)) } diff --git a/crates/iceberg/src/puffin/writer.rs b/crates/iceberg/src/puffin/writer.rs index 0437bd5bf0..e6882a4844 100644 --- a/crates/iceberg/src/puffin/writer.rs +++ b/crates/iceberg/src/puffin/writer.rs @@ -23,7 +23,7 @@ use super::validate_puffin_compression; use crate::Result; use crate::compression::CompressionCodec; use crate::encryption::EncryptedOutputFile; -use crate::io::{FileWrite, OutputFile}; +use crate::io::{FileMetadata as IoFileMetadata, FileWrite, OutputFile}; use crate::puffin::blob::Blob; use crate::puffin::metadata::{BlobMetadata, FileMetadata, Flag}; @@ -117,12 +117,11 @@ impl PuffinWriter { Ok(()) } - /// Finalizes the Puffin file - pub async fn close(mut self) -> Result<()> { + /// Finalizes the Puffin file and returns its stored size. + pub async fn close(mut self) -> Result { self.write_header_once().await?; self.write_footer().await?; - self.writer.close().await?; - Ok(()) + self.writer.close().await } async fn write(&mut self, bytes: Bytes) -> Result<()> { @@ -409,7 +408,7 @@ mod tests { for blob in blobs.clone() { writer.add(blob, CompressionCodec::None).await.unwrap(); } - writer.close().await.unwrap(); + let file_metadata = writer.close().await.unwrap(); // The ciphertext on disk must not equal a plaintext puffin file. let raw = file_io.new_input(path).unwrap().read().await.unwrap(); @@ -419,8 +418,10 @@ mod tests { ); // Read back through the decrypting reader over plaintext offsets. + let key_metadata = encrypted_output.key_metadata_with_saved_file_metadata(&file_metadata); + assert_eq!(key_metadata.file_length(), Some(raw.len() as u64)); let encrypted_input = - EncryptedInputFile::new(file_io.new_input(path).unwrap(), key_metadata()); + EncryptedInputFile::new(file_io.new_input(path).unwrap(), key_metadata); let reader = PuffinReader::new_from_encrypted(encrypted_input) .await .unwrap(); diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index b80567a683..15608a63f8 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -16,8 +16,6 @@ // under the License. use std::cmp::min; -use std::future::Future; -use std::pin::Pin; use apache_avro::{Writer as AvroWriter, to_value}; use bytes::Bytes; @@ -30,7 +28,7 @@ use super::{ }; use crate::encryption::EncryptedOutputFile; use crate::error::{Result, invalid_data}; -use crate::io::{FileWrite, OutputFile}; +use crate::io::{FileMetadata, FileWrite, OutputFile}; use crate::spec::manifest::_serde::{ManifestEntryV1, ManifestEntryV2}; use crate::spec::manifest::{manifest_schema_v1, manifest_schema_v2}; use crate::spec::{ @@ -42,14 +40,38 @@ use crate::spec::{ /// with the actual snapshot ID before it is committed. const UNASSIGNED_SNAPSHOT_ID: i64 = -1; -type WriterFuture = Pin>> + Send>>; +/// Retains the output until close provides the size required by encrypted key metadata. +pub(crate) enum ManifestOutput { + Plain(OutputFile), + Encrypted(EncryptedOutputFile), +} + +impl ManifestOutput { + async fn writer(&self) -> Result> { + match self { + Self::Plain(output) => output.writer().await, + Self::Encrypted(output) => output.writer().await, + } + } + + fn encoded_key_metadata(&self, file_metadata: &FileMetadata) -> Result>> { + match self { + Self::Plain(_) => Ok(None), + Self::Encrypted(output) => Ok(Some( + output + .key_metadata_with_saved_file_metadata(file_metadata) + .encode()? + .into_vec(), + )), + } + } +} /// The builder used to create a [`ManifestWriter`]. pub struct ManifestWriterBuilder { - writer_future: WriterFuture, + output: ManifestOutput, location: String, snapshot_id: Option, - key_metadata: Option>, schema: SchemaRef, partition_spec: PartitionSpec, } @@ -64,10 +86,9 @@ impl ManifestWriterBuilder { ) -> Self { let location = output.location().to_owned(); Self { - writer_future: Box::pin(async move { output.writer().await }), + output: ManifestOutput::Plain(output), location, snapshot_id, - key_metadata: None, schema, partition_spec, } @@ -83,12 +104,10 @@ impl ManifestWriterBuilder { partition_spec: PartitionSpec, ) -> Result { let location = encrypted_output.location().to_owned(); - let key_metadata = Some(encrypted_output.key_metadata().encode()?.to_vec()); Ok(Self { - writer_future: Box::pin(async move { encrypted_output.writer().await }), + output: ManifestOutput::Encrypted(encrypted_output), location, snapshot_id, - key_metadata, schema, partition_spec, }) @@ -103,14 +122,7 @@ impl ManifestWriterBuilder { .format_version(FormatVersion::V1) .content(ManifestContentType::Data) .build(); - ManifestWriter::new( - self.writer_future, - self.location, - self.snapshot_id, - self.key_metadata, - metadata, - None, - ) + ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None) } /// Build a [`ManifestWriter`] for format version 2, data content. @@ -123,14 +135,7 @@ impl ManifestWriterBuilder { .content(ManifestContentType::Data) .build(); - ManifestWriter::new( - self.writer_future, - self.location, - self.snapshot_id, - self.key_metadata, - metadata, - None, - ) + ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None) } /// Build a [`ManifestWriter`] for format version 2, deletes content. @@ -142,14 +147,7 @@ impl ManifestWriterBuilder { .format_version(FormatVersion::V2) .content(ManifestContentType::Deletes) .build(); - ManifestWriter::new( - self.writer_future, - self.location, - self.snapshot_id, - self.key_metadata, - metadata, - None, - ) + ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None) } /// Build a [`ManifestWriter`] for format version 2, data content. @@ -162,10 +160,9 @@ impl ManifestWriterBuilder { .content(ManifestContentType::Data) .build(); ManifestWriter::new( - self.writer_future, + self.output, self.location, self.snapshot_id, - self.key_metadata, metadata, // First row id is assigned by the [`ManifestListWriter`] when the manifest // is added to the list. @@ -182,20 +179,13 @@ impl ManifestWriterBuilder { .format_version(FormatVersion::V3) .content(ManifestContentType::Deletes) .build(); - ManifestWriter::new( - self.writer_future, - self.location, - self.snapshot_id, - self.key_metadata, - metadata, - None, - ) + ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None) } } /// A manifest writer. pub struct ManifestWriter { - writer_future: WriterFuture, + output: ManifestOutput, location: String, snapshot_id: Option, @@ -210,8 +200,6 @@ pub struct ManifestWriter { min_seq_num: Option, - key_metadata: Option>, - manifest_entries: Vec, metadata: ManifestMetadata, @@ -220,15 +208,14 @@ pub struct ManifestWriter { impl ManifestWriter { /// Create a new manifest writer. pub(crate) fn new( - writer_future: WriterFuture, + output: ManifestOutput, location: String, snapshot_id: Option, - key_metadata: Option>, metadata: ManifestMetadata, first_row_id: Option, ) -> Self { Self { - writer_future, + output, location, snapshot_id, added_files: 0, @@ -239,7 +226,6 @@ impl ManifestWriter { deleted_rows: 0, first_row_id, min_seq_num: None, - key_metadata, manifest_entries: Vec::new(), metadata, } @@ -497,14 +483,15 @@ impl ManifestWriter { } let content = avro_writer.into_inner()?; - let length = content.len(); - let mut writer = self.writer_future.await?; + let mut writer = self.output.writer().await?; writer.write(Bytes::from(content)).await?; - writer.close().await?; + let file_metadata = writer.close().await?; + let key_metadata = self.output.encoded_key_metadata(&file_metadata)?; Ok(ManifestFile { manifest_path: self.location, - manifest_length: length as i64, + // Manifest lengths are on-disk sizes, including encryption overhead. + manifest_length: file_metadata.size.try_into()?, partition_spec_id: self.metadata.partition_spec.spec_id(), content: self.metadata.content, // sequence_number and min_sequence_number with UNASSIGNED_SEQUENCE_NUMBER will be replace with @@ -519,7 +506,7 @@ impl ManifestWriter { existing_rows_count: Some(self.existing_rows), deleted_rows_count: Some(self.deleted_rows), partitions: Some(partition_summary), - key_metadata: self.key_metadata, + key_metadata, first_row_id: self.first_row_id, }) } @@ -822,9 +809,9 @@ mod tests { assert_eq!(manifest_file.content, ManifestContentType::Deletes); // Read back the manifest file - let actual_manifest = - Manifest::parse_avro(fs::read(&path).expect("read_file must succeed").as_slice()) - .unwrap(); + let bytes = fs::read(&path).expect("read_file must succeed"); + assert_eq!(manifest_file.manifest_length, bytes.len() as i64); + let actual_manifest = Manifest::parse_avro(&bytes).unwrap(); // Verify the content type is correctly preserved as Deletes assert_eq!( diff --git a/crates/iceberg/src/spec/manifest_list/manifest_file.rs b/crates/iceberg/src/spec/manifest_list/manifest_file.rs index 341748a416..24df6d4dca 100644 --- a/crates/iceberg/src/spec/manifest_list/manifest_file.rs +++ b/crates/iceberg/src/spec/manifest_list/manifest_file.rs @@ -286,12 +286,15 @@ mod test { let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") .unwrap() .with_aad_prefix(b"test-aad-prefix!"); - let encoded_key_metadata = key_metadata.encode().unwrap().to_vec(); - let io = FileIO::new_with_memory(); let path = "memory:///test/encrypted_manifest.avro"; - let manifest_file = write_encrypted_manifest(&io, path, key_metadata).await; - assert_eq!(manifest_file.key_metadata, Some(encoded_key_metadata)); + let manifest_file = write_encrypted_manifest(&io, path, key_metadata.clone()).await; + let size = io.new_input(path).unwrap().metadata().await.unwrap().size; + assert_eq!(manifest_file.manifest_length, size as i64); + assert_eq!( + StandardKeyMetadata::decode(manifest_file.key_metadata.as_ref().unwrap()).unwrap(), + key_metadata.with_file_length(size) + ); let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap(); assert_eq!(manifest.entries().len(), 1); @@ -318,7 +321,8 @@ mod test { // returning garbage. let wrong_key_metadata = StandardKeyMetadata::try_new(b"fedcba9876543210") .unwrap() - .with_aad_prefix(b"test-aad-prefix!"); + .with_aad_prefix(b"test-aad-prefix!") + .with_file_length(manifest_file.manifest_length as u64); manifest_file.key_metadata = Some(wrong_key_metadata.encode().unwrap().to_vec()); let err = ManifestReader::new(io) @@ -343,7 +347,8 @@ mod test { // so GCM authentication must fail even though the key is right. let wrong_aad_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") .unwrap() - .with_aad_prefix(b"wrong-aad-prefix"); + .with_aad_prefix(b"wrong-aad-prefix") + .with_file_length(manifest_file.manifest_length as u64); manifest_file.key_metadata = Some(wrong_aad_metadata.encode().unwrap().to_vec()); let err = ManifestReader::new(io) diff --git a/crates/iceberg/src/spec/manifest_list/writer.rs b/crates/iceberg/src/spec/manifest_list/writer.rs index 24613770d2..5ce2cfb259 100644 --- a/crates/iceberg/src/spec/manifest_list/writer.rs +++ b/crates/iceberg/src/spec/manifest_list/writer.rs @@ -26,7 +26,7 @@ use super::_const_schema::{ use super::_serde::{ManifestFileV1, ManifestFileV2, ManifestFileV3}; use super::{FormatVersion, ManifestContentType, ManifestFile, UNASSIGNED_SEQUENCE_NUMBER}; use crate::error::{Result, invalid_data}; -use crate::io::FileWrite; +use crate::io::{FileMetadata, FileWrite}; use crate::{Error, ErrorKind}; /// A manifest list writer. @@ -196,12 +196,11 @@ impl ManifestListWriter { Ok(()) } - /// Write the manifest list to the output file. - pub async fn close(mut self) -> Result<()> { + /// Write the manifest list and return its stored size. + pub async fn close(mut self) -> Result { let data = self.avro_writer.into_inner()?; self.writer.write(Bytes::from(data)).await?; - self.writer.close().await?; - Ok(()) + self.writer.close().await } /// Assign sequence numbers to manifest if they are unassigned @@ -601,7 +600,6 @@ mod test { let path = "memory:///manifest_list_v3_encrypted.avro"; let encrypted_output = mgr.encrypt(file_io.new_output(path).unwrap()); - let key_metadata = encrypted_output.key_metadata().clone(); let snapshot_id = 9_000_000_000_000_001i64; let seq_num = 7i64; @@ -636,7 +634,7 @@ mod test { writer .add_manifests(expected.entries.clone().into_iter()) .unwrap(); - writer.close().await.unwrap(); + let file_metadata = writer.close().await.unwrap(); let raw_bytes = file_io.new_input(path).unwrap().read().await.unwrap(); assert!( @@ -644,6 +642,8 @@ mod test { "raw bytes should be ciphertext, not parseable as Avro" ); + let key_metadata = encrypted_output.key_metadata_with_saved_file_metadata(&file_metadata); + assert_eq!(key_metadata.file_length(), Some(raw_bytes.len() as u64)); let plaintext = EncryptedInputFile::new(file_io.new_input(path).unwrap(), key_metadata) .read() .await diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 872a1d89c1..3260f9ffa2 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -420,6 +420,25 @@ mod tests { }) .expect("a fast append should emit an AddSnapshot update"); + let manifest_list_key_metadata = table + .encryption_manager() + .unwrap() + .decrypt_manifest_list_key_metadata(new_snapshot.encryption_key_id().unwrap()) + .await + .unwrap(); + let manifest_list_size = table + .file_io() + .new_input(new_snapshot.manifest_list()) + .unwrap() + .metadata() + .await + .unwrap() + .size; + assert_eq!( + manifest_list_key_metadata.file_length(), + Some(manifest_list_size) + ); + let manifest_list = table .manifest_list_reader(&new_snapshot) .load() @@ -436,8 +455,18 @@ mod tests { .key_metadata .as_ref() .expect("encrypted manifest must record key metadata"); - StandardKeyMetadata::decode(key_metadata_bytes) + let key_metadata = StandardKeyMetadata::decode(key_metadata_bytes) .expect("recorded key metadata must decode as StandardKeyMetadata"); + let manifest_size = table + .file_io() + .new_input(&manifest_file.manifest_path) + .unwrap() + .metadata() + .await + .unwrap() + .size; + assert_eq!(key_metadata.file_length(), Some(manifest_size)); + assert_eq!(manifest_file.manifest_length, manifest_size as i64); // The reader self-decrypts using the recorded key metadata and must // recover the entry we appended. Because the read goes through the diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 43e8b25d5e..a0ce1fba14 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -444,13 +444,13 @@ impl<'a> SnapshotProducer<'a> { .file_io() .new_output(manifest_list_path.clone())?; - let (writer, encryption_key_id) = match self.table.encryption_manager() { + let (writer, encrypted_output) = match self.table.encryption_manager() { Some(em) => { let encrypted_output = em.encrypt(raw_output); - let key_id = em - .encrypt_manifest_list_key_metadata(encrypted_output.key_metadata()) - .await?; - (encrypted_output.writer().await?, Some(key_id)) + ( + encrypted_output.writer().await?, + Some((em.clone(), encrypted_output)), + ) } None => (raw_output.writer().await?, None), }; @@ -485,7 +485,16 @@ impl<'a> SnapshotProducer<'a> { manifest_list_writer.add_manifests(new_manifests.into_iter())?; let writer_next_row_id = manifest_list_writer.next_row_id(); - manifest_list_writer.close().await?; + let file_metadata = manifest_list_writer.close().await?; + let encryption_key_id = match encrypted_output { + Some((em, encrypted_output)) => Some( + em.encrypt_manifest_list_key_metadata( + &encrypted_output.key_metadata_with_saved_file_metadata(&file_metadata), + ) + .await?, + ), + None => None, + }; let commit_ts = chrono::Utc::now().timestamp_millis(); let new_snapshot = Snapshot::builder() diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index b874cb95a9..7cdc488889 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -732,9 +732,11 @@ impl ArrowAsyncFileWriter for AsyncFileWriter { fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> { Box::pin(async { + // TODO(encryption): retain the stored file size in data-file key metadata. self.0 .close() .await + .map(|_| ()) .map_err(|err| parquet::errors::ParquetError::External(Box::new(err))) }) } diff --git a/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidEncryption.json b/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidEncryption.json index 0dc0aabc17..1c096417fb 100644 --- a/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidEncryption.json +++ b/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidEncryption.json @@ -63,7 +63,7 @@ }, { "key-id": "1fb95f04-7f98-4b1f-a08f-e810cfb907ff", - "encrypted-key-metadata": "scAmTJzgxf7vDQBetLMWSKnZmOljEzoitb6H90HQ/wjDz5/naIkg44PkAg+Ldl6RBP0FBcoS1bfBDVLToUbdrVA=", + "encrypted-key-metadata": "XcrqLxxyf9a7FAFXM63u3MEH8CSWqh6g1H8fJD2/6zx0EjcPzPaP/5AtRpO2s6p2isV8s7a4m77zj+GIqNmIpbA2Xw==", "encrypted-by-id": "5b29941b-b3b4-4d57-8758-a9d1bf340b39" } ], diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 2ed5a60cb1..fe485cb915 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -551,7 +551,7 @@ impl Storage for OpenDalStorage { async fn writer(&self, path: &str) -> Result> { let (op, relative_path) = self.create_operator(&path)?; - Ok(Box::new(OpenDalWriter( + Ok(Box::new(OpenDalWriter::new( op.writer(relative_path).await.map_err(from_opendal_error)?, ))) } @@ -635,21 +635,51 @@ impl FileRead for OpenDalReader { } /// Wrapper around `opendal::Writer` that implements `FileWrite`. -pub(crate) struct OpenDalWriter(pub(crate) opendal::Writer); +pub(crate) struct OpenDalWriter { + inner: opendal::Writer, + bytes_written: u64, +} + +impl OpenDalWriter { + pub(crate) fn new(inner: opendal::Writer) -> Self { + Self { + inner, + bytes_written: 0, + } + } +} #[async_trait] impl FileWrite for OpenDalWriter { async fn write(&mut self, bs: Bytes) -> Result<()> { - Ok(opendal::Writer::write(&mut self.0, bs) + let len = bs.len() as u64; + opendal::Writer::write(&mut self.inner, bs) .await - .map_err(from_opendal_error)?) + .map_err(from_opendal_error)?; + self.bytes_written += len; + Ok(()) } - async fn close(&mut self) -> Result<()> { - let _ = opendal::Writer::close(&mut self.0) + async fn close(&mut self) -> Result { + let metadata = opendal::Writer::close(&mut self.inner) .await .map_err(from_opendal_error)?; - Ok(()) + + // Object stores may omit the size (reported as 0); validate only a reported nonzero size. + let reported_size = metadata.content_length(); + if reported_size != 0 && reported_size != self.bytes_written { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Wrote {} bytes but storage reports {reported_size}", + self.bytes_written + ), + )); + } + + Ok(FileMetadata { + size: self.bytes_written, + }) } } @@ -695,6 +725,38 @@ mod tests { assert_eq!(op.info().scheme().to_string(), "memory"); } + #[cfg(feature = "opendal-memory")] + #[tokio::test] + async fn test_writer_close_returns_stored_size() { + use iceberg::encryption::{EncryptedOutputFile, StandardKeyMetadata}; + + // Note: the memory service does report a content length, so this only pins the happy + // path. The counter in `OpenDalWriter` is what covers services that don't, such as S3. + let storage = Arc::new(OpenDalStorage::Memory(default_memory_operator())); + let path = "memory:///stored-size"; + for plaintext in [ + Bytes::new(), + Bytes::from_static(b"test data"), + Bytes::from(vec![7; 3 * 1024]), + ] { + let mut writer = storage.writer(path).await.unwrap(); + for chunk in plaintext.chunks(1024) { + writer.write(Bytes::copy_from_slice(chunk)).await.unwrap(); + } + let metadata = writer.close().await.unwrap(); + assert_eq!(metadata.size, plaintext.len() as u64); + assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size); + + let output = EncryptedOutputFile::new( + OutputFile::new(storage.clone(), path.to_string()), + StandardKeyMetadata::try_new(b"0123456789abcdef").unwrap(), + ); + let metadata = output.write(plaintext.clone()).await.unwrap(); + assert!(metadata.size > plaintext.len() as u64); + assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size); + } + } + #[cfg(feature = "opendal-memory")] #[test] fn test_relativize_path_memory() { From 119fad92c891c95a7819d87e4195407b3f7c0578 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Thu, 24 Sep 2026 18:51:51 +0000 Subject: [PATCH 13/63] ci: replace MinIO with RustFS for integration tests (#3273) * ci: replace MinIO with RustFS for integration tests `quay.io/minio/minio` no longer allows anonymous pulls, so `make docker-up` fails and the integration tests cannot start. DataFusion hit the same failure and switched to RustFS in apache/datafusion#25706. PyIceberg moved to RustFS earlier in apache/iceberg-python#3928. Run `rustfs/rustfs:1.0.0` as the shared S3 service instead. It keeps the `admin`/`password` credentials, the 9000/9001 ports, and virtual-hosted-style access through `RUSTFS_SERVER_DOMAINS` and the bucket aliases. The healthcheck uses `/health/ready`, which waits for storage and IAM. The `mc` bucket setup becomes `curl --aws-sigv4` run from the RustFS image, and the public bucket policy is dropped because no test reads anonymously. Point the REST fixture, HMS, and Spark at `http://rustfs:9000`, and rename the MinIO-specific test helpers and the `ICEBERG_TEST_MINIO_ENDPOINT` override to RustFS. Closes #3272 * ci: use vendor-neutral object-store naming for S3 test service Rename the RustFS service, hostname, bucket aliases, test helpers and env var to a generic object-store name (matching PyIceberg), so future backend swaps only need an image change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * ci: fail fast and gate readiness on test bucket creation Treat any non-200/409 response as a failure instead of ignoring it, and add a healthcheck so `docker compose up --wait` blocks until the buckets exist. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * ci: remove orphan containers on docker-up Renaming the minio service leaves stale containers holding port 9000 for anyone with a previously started stack; clean them up automatically. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * ci: make create-buckets a one-shot job RustFS returns 200 when re-creating an existing bucket, so plain chained curl -f calls are idempotent. Replace the status-code parsing, marker-file healthcheck and tail with a one-shot container, and gate spark-iceberg on its successful completion. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: ovoievodin Co-authored-by: Kevin Liu Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Makefile | 2 +- crates/catalog/hms/tests/hms_catalog_test.rs | 6 +- crates/catalog/loader/tests/common/mod.rs | 11 ++-- crates/integration_tests/src/lib.rs | 6 +- .../storage/opendal/tests/file_io_s3_test.rs | 20 +++---- .../opendal/tests/resolving_storage_test.rs | 16 ++--- crates/test_utils/src/lib.rs | 14 ++--- dev/docker-compose.yaml | 58 ++++++++----------- dev/hms/core-site.xml | 2 +- dev/spark/spark-defaults.conf | 2 +- website/src/reference/container-runtimes.md | 2 +- 11 files changed, 65 insertions(+), 74 deletions(-) diff --git a/Makefile b/Makefile index d27893390a..5371c42287 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ site: install-mdbook # Docker targets for integration tests docker-up: - docker compose -f dev/docker-compose.yaml up -d --build --wait + docker compose -f dev/docker-compose.yaml up -d --build --wait --remove-orphans docker-down: docker compose -f dev/docker-compose.yaml down -v --remove-orphans --timeout 0 diff --git a/crates/catalog/hms/tests/hms_catalog_test.rs b/crates/catalog/hms/tests/hms_catalog_test.rs index d0e6486ad8..b64184ce77 100644 --- a/crates/catalog/hms/tests/hms_catalog_test.rs +++ b/crates/catalog/hms/tests/hms_catalog_test.rs @@ -33,7 +33,7 @@ use iceberg_catalog_hms::{ HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED, }; use iceberg_storage_opendal::OpenDalStorageFactory; -use iceberg_test_utils::{get_hms_endpoint, get_minio_endpoint, set_up}; +use iceberg_test_utils::{get_hms_endpoint, get_object_store_endpoint, set_up}; use tokio::time::sleep; use tracing::info; @@ -43,7 +43,7 @@ async fn get_catalog() -> HmsCatalog { set_up(); let hms_endpoint = get_hms_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let props = HashMap::from([ (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), @@ -55,7 +55,7 @@ async fn get_catalog() -> HmsCatalog { HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/catalog/loader/tests/common/mod.rs b/crates/catalog/loader/tests/common/mod.rs index 2b5033e6f2..0b7e22cea1 100644 --- a/crates/catalog/loader/tests/common/mod.rs +++ b/crates/catalog/loader/tests/common/mod.rs @@ -50,7 +50,8 @@ use iceberg_catalog_sql::{ }; use iceberg_storage_opendal::OpenDalStorageFactory; use iceberg_test_utils::{ - get_glue_endpoint, get_hms_endpoint, get_minio_endpoint, get_rest_catalog_endpoint, set_up, + get_glue_endpoint, get_hms_endpoint, get_object_store_endpoint, get_rest_catalog_endpoint, + set_up, }; use sqlx::migrate::MigrateDatabase; use tempfile::TempDir; @@ -230,7 +231,7 @@ async fn rest_catalog(kms_client_factory: Arc) -> RestCata async fn glue_catalog(kms_client_factory: Arc) -> GlueCatalog { let glue_endpoint = get_glue_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let props = HashMap::from([ (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), @@ -239,7 +240,7 @@ async fn glue_catalog(kms_client_factory: Arc) -> GlueCata "my_secret_key".to_string(), ), (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), @@ -279,7 +280,7 @@ async fn glue_catalog(kms_client_factory: Arc) -> GlueCata async fn hms_catalog(kms_client_factory: Arc) -> HmsCatalog { let hms_endpoint = get_hms_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let props = HashMap::from([ (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), @@ -291,7 +292,7 @@ async fn hms_catalog(kms_client_factory: Arc) -> HmsCatalo HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/integration_tests/src/lib.rs b/crates/integration_tests/src/lib.rs index feafa3ae9f..c137b46f86 100644 --- a/crates/integration_tests/src/lib.rs +++ b/crates/integration_tests/src/lib.rs @@ -22,7 +22,7 @@ use iceberg::io::{ S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, }; use iceberg_catalog_rest::REST_CATALOG_PROP_URI; -use iceberg_test_utils::{get_minio_endpoint, get_rest_catalog_endpoint, set_up}; +use iceberg_test_utils::{get_object_store_endpoint, get_rest_catalog_endpoint, set_up}; /// Global test fixture that uses environment-based configuration. /// This assumes Docker containers are started externally (e.g., via `make docker-up`). @@ -39,11 +39,11 @@ impl GlobalTestFixture { set_up(); let rest_endpoint = get_rest_catalog_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let catalog_config = HashMap::from([ (REST_CATALOG_PROP_URI.to_string(), rest_endpoint), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 6b57f86607..ac6342dddb 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -32,19 +32,19 @@ mod tests { use iceberg_storage_opendal::{ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, }; - use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; use reqsign_core::Context; async fn get_file_io() -> FileIO { set_up(); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: None, })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_ACCESS_KEY_ID, "admin".to_string()), (S3_SECRET_ACCESS_KEY, "password".to_string()), (S3_REGION, "us-east-1".to_string()), @@ -138,7 +138,7 @@ mod tests { Self { credential } } - fn new_minio() -> Self { + fn new_object_store() -> Self { Self::new(Some(AwsCredential { access_key_id: "admin".to_string(), secret_access_key: "password".to_string(), @@ -162,7 +162,7 @@ mod tests { #[test] fn test_custom_aws_credential_loader_instantiation() { // Test creating CustomAwsCredentialLoader with mock loader - let mock_loader = MockCredentialLoader::new_minio(); + let mock_loader = MockCredentialLoader::new_object_store(); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); // Test that the loader can be used in FileIOBuilder with OpenDalStorageFactory @@ -182,17 +182,17 @@ mod tests { let _file_io = get_file_io().await; // Create a mock credential loader - let mock_loader = MockCredentialLoader::new_minio(); + let mock_loader = MockCredentialLoader::new_object_store(); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); // Build FileIO with custom credential loader via OpenDalStorageFactory let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: Some(custom_loader), })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) @@ -213,14 +213,14 @@ mod tests { let mock_loader = MockCredentialLoader::new(None); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); // Build FileIO with custom credential loader via OpenDalStorageFactory let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: Some(custom_loader), })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index ba94b8332d..bcc3d51842 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -33,16 +33,16 @@ mod tests { S3_SECRET_ACCESS_KEY, }; use iceberg_storage_opendal::OpenDalResolvingStorageFactory; - use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; fn get_resolving_file_io() -> iceberg::io::FileIO { set_up(); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_ACCESS_KEY_ID, "admin".to_string()), (S3_SECRET_ACCESS_KEY, "password".to_string()), (S3_REGION, "us-east-1".to_string()), @@ -268,9 +268,9 @@ mod tests { use reqsign_core::Context; #[derive(Debug)] - struct MinioCredentialLoader; + struct ObjectStoreCredentialLoader; - impl ProvideCredential for MinioCredentialLoader { + impl ProvideCredential for ObjectStoreCredentialLoader { type Credential = AwsCredential; async fn provide_credential( @@ -287,14 +287,14 @@ mod tests { } set_up(); - let minio_endpoint = get_minio_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let factory = OpenDalResolvingStorageFactory::new() - .with_s3_credential_loader(CustomAwsCredentialLoader::new(MinioCredentialLoader)); + .with_s3_credential_loader(CustomAwsCredentialLoader::new(ObjectStoreCredentialLoader)); let file_io = FileIOBuilder::new(Arc::new(factory)) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index e44d96c385..b64e462696 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -37,24 +37,24 @@ mod common { } // Environment variable names for service endpoints - pub const ENV_MINIO_ENDPOINT: &str = "ICEBERG_TEST_MINIO_ENDPOINT"; + pub const ENV_OBJECT_STORE_ENDPOINT: &str = "ICEBERG_TEST_OBJECT_STORE_ENDPOINT"; pub const ENV_REST_CATALOG_ENDPOINT: &str = "ICEBERG_TEST_REST_ENDPOINT"; pub const ENV_HMS_ENDPOINT: &str = "ICEBERG_TEST_HMS_ENDPOINT"; pub const ENV_GLUE_ENDPOINT: &str = "ICEBERG_TEST_GLUE_ENDPOINT"; pub const ENV_GCS_ENDPOINT: &str = "ICEBERG_TEST_GCS_ENDPOINT"; // Default ports matching dev/docker-compose.yaml - pub const DEFAULT_MINIO_PORT: u16 = 9000; + pub const DEFAULT_OBJECT_STORE_PORT: u16 = 9000; pub const DEFAULT_REST_CATALOG_PORT: u16 = 8181; pub const DEFAULT_HMS_PORT: u16 = 9083; pub const DEFAULT_GLUE_PORT: u16 = 5001; pub const DEFAULT_GCS_PORT: u16 = 4443; - /// Returns the MinIO S3-compatible endpoint. - /// Checks ICEBERG_TEST_MINIO_ENDPOINT env var, otherwise returns localhost default. - pub fn get_minio_endpoint() -> String { - std::env::var(ENV_MINIO_ENDPOINT) - .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_MINIO_PORT}")) + /// Returns the S3-compatible object store endpoint. + /// Checks ICEBERG_TEST_OBJECT_STORE_ENDPOINT env var, otherwise returns localhost default. + pub fn get_object_store_endpoint() -> String { + std::env::var(ENV_OBJECT_STORE_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_OBJECT_STORE_PORT}")) } /// Returns the REST catalog endpoint. diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 16c999cb9a..cb89e3042d 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -25,53 +25,43 @@ networks: services: # ============================================================================= - # MinIO - S3-compatible storage (shared by all tests) + # Object store - S3-compatible storage backed by RustFS (shared by all tests) # ============================================================================= - minio: - image: quay.io/minio/minio:RELEASE.2025-05-24T17-08-30Z + object-store: + image: rustfs/rustfs:1.0.0 environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - hostname: minio + - RUSTFS_ACCESS_KEY=admin + - RUSTFS_SECRET_KEY=password + - RUSTFS_SERVER_DOMAINS=object-store + hostname: object-store networks: iceberg_test: # Add aliases for virtual-hosted style bucket access aliases: - - icebergdata.minio - - warehouse.minio - - bucket1.minio + - icebergdata.object-store + - warehouse.object-store + - bucket1.object-store ports: - "9000:9000" - "9001:9001" - command: ["server", "/data", "--console-address", ":9001"] healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "curl", "-f", "http://localhost:9000/health/ready"] interval: 5s timeout: 5s retries: 5 - # MinIO client - creates buckets for tests - mc: + # One-shot job that creates buckets for tests. Re-creating an existing bucket + # returns 200, so this is safe to re-run. + create-buckets: depends_on: - minio: + object-store: condition: service_healthy - image: quay.io/minio/mc:RELEASE.2025-05-21T01-59-54Z - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 + image: rustfs/rustfs:1.0.0 entrypoint: > /bin/sh -c " - /usr/bin/mc alias set minio http://minio:9000 admin password; - /usr/bin/mc mb --ignore-existing minio/icebergdata; - /usr/bin/mc mb --ignore-existing minio/warehouse; - /usr/bin/mc mb --ignore-existing minio/bucket1; - /usr/bin/mc policy set public minio/icebergdata; - /usr/bin/mc policy set public minio/warehouse; - /usr/bin/mc policy set public minio/bucket1; - echo 'Buckets created successfully'; - tail -f /dev/null + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/icebergdata && + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/warehouse && + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/bucket1 " networks: iceberg_test: @@ -89,9 +79,9 @@ services: - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest.db?journal_mode=WAL - CATALOG_WAREHOUSE=s3://icebergdata/demo - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://minio:9000 + - CATALOG_S3_ENDPOINT=http://object-store:9000 depends_on: - minio: + object-store: condition: service_healthy networks: iceberg_test: @@ -113,7 +103,7 @@ services: dockerfile: Dockerfile platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} depends_on: - minio: + object-store: condition: service_healthy networks: iceberg_test: @@ -170,8 +160,8 @@ services: depends_on: rest: condition: service_healthy - minio: - condition: service_healthy + create-buckets: + condition: service_completed_successfully environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password diff --git a/dev/hms/core-site.xml b/dev/hms/core-site.xml index f23efa8885..d88ef93570 100644 --- a/dev/hms/core-site.xml +++ b/dev/hms/core-site.xml @@ -30,7 +30,7 @@ fs.s3a.endpoint - http://minio:9000 + http://object-store:9000 fs.s3a.access.key diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index b576f624b9..a02640bbdd 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -23,7 +23,7 @@ spark.sql.catalog.rest.type rest spark.sql.catalog.rest.uri http://rest:8181 spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.rest.warehouse s3://warehouse/rest/ -spark.sql.catalog.rest.s3.endpoint http://minio:9000 +spark.sql.catalog.rest.s3.endpoint http://object-store:9000 spark.sql.catalog.rest.cache-enabled false spark.sql.defaultCatalog rest diff --git a/website/src/reference/container-runtimes.md b/website/src/reference/container-runtimes.md index 7b4ac38511..412aa70053 100644 --- a/website/src/reference/container-runtimes.md +++ b/website/src/reference/container-runtimes.md @@ -19,7 +19,7 @@ # Container Runtimes -Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for MinIO and various catalogs. You can use any of the following container runtimes. +Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for an S3-compatible object store (RustFS) and various catalogs. You can use any of the following container runtimes. ## Docker Desktop From 9a2111a914e9b17fdc039700186056ec14fae606 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Fri, 25 Sep 2026 21:32:09 +0000 Subject: [PATCH 14/63] fix(spec): format pre-epoch timestamptz values (#2892) * fix(spec): format pre-epoch timestamptz values * fix(spec): strengthen pre-epoch timestamptz regression * style: format temporal conversion * test(spec): match chrono fractional timestamp display * test(spec): match trimmed timestamp fraction --- crates/iceberg/src/spec/values/temporal.rs | 12 ++++---- crates/iceberg/src/spec/values/tests.rs | 32 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/crates/iceberg/src/spec/values/temporal.rs b/crates/iceberg/src/spec/values/temporal.rs index 80aadeb659..76b2c897e1 100644 --- a/crates/iceberg/src/spec/values/temporal.rs +++ b/crates/iceberg/src/spec/values/temporal.rs @@ -98,15 +98,15 @@ pub(crate) mod timestamptz { } pub(crate) fn microseconds_to_datetimetz(micros: i64) -> DateTime { - let (secs, rem) = (micros / 1_000_000, micros % 1_000_000); - - DateTime::from_timestamp(secs, rem as u32 * 1_000).unwrap() + // Returns None only outside chrono's range (~year -262144 to +262142); + // unreachable for any realistic Iceberg timestamp. + DateTime::from_timestamp_micros(micros).expect( + "timestamp micros outside chrono's representable range (~year -262144..+262142)", + ) } pub(crate) fn nanoseconds_to_datetimetz(nanos: i64) -> DateTime { - let (secs, rem) = (nanos / 1_000_000_000, nanos % 1_000_000_000); - - DateTime::from_timestamp(secs, rem as u32).unwrap() + DateTime::from_timestamp_nanos(nanos) } /// Nanoseconds since the Unix epoch, or `None` if outside the representable `i64` range diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index 97b2f6b325..ff51f318e3 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -1124,6 +1124,38 @@ fn test_parse_timestamptz() { ); } +#[test] +fn test_pre_epoch_timestamptz_formatting() { + assert_eq!( + Datum::timestamptz_micros(-1).to_string(), + "1969-12-31 23:59:59.999999 UTC" + ); + assert_eq!( + Datum::timestamptz_nanos(-1).to_string(), + "1969-12-31 23:59:59.999999999 UTC" + ); + assert_eq!( + Datum::timestamptz_micros(-1_500_000).to_string(), + "1969-12-31 23:59:58.500 UTC" + ); + + check_json_serde( + r#""1969-12-31T23:59:59.999999+00:00""#, + Literal::long(-1), + &Primitive(PrimitiveType::Timestamptz), + ); + check_json_serde( + r#""1969-12-31T23:59:58.500+00:00""#, + Literal::long(-1_500_000), + &Primitive(PrimitiveType::Timestamptz), + ); + check_json_serde( + r#""1969-12-31T23:59:59.999999999+00:00""#, + Literal::long(-1), + &Primitive(PrimitiveType::TimestamptzNs), + ); +} + #[test] fn test_datum_ser_deser() { let test_fn = |datum: Datum| { From 615af68da85f2cc6e5139aa180c939214de91d32 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Sat, 26 Sep 2026 09:28:36 +0000 Subject: [PATCH 15/63] fix(expr): decode BYTE_ARRAY page-index bounds by field type (#3247) * fix(expr): decode BYTE_ARRAY page-index bounds by field type The page index evaluator decoded every BYTE_ARRAY min/max bound as a UTF-8 string via String::from_utf8(..).unwrap(), panicking during scan pruning whenever a binary (or byte-encoded decimal) column carried a predicate. Convert the bound according to the field's primitive type instead: string, binary, or decimal. * PR feedback * Round2 feedback * Fix trace --- .../src/expr/visitors/page_index_evaluator.rs | 376 ++++++++++++++++-- 1 file changed, 343 insertions(+), 33 deletions(-) diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index bd901eda5f..37d9c6a233 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -191,6 +191,7 @@ impl<'a> PageIndexEvaluator<'a> { }; let Some(page_filter) = Self::apply_predicate_to_column_index( + field_id, predicate, field_type, column_index, @@ -238,6 +239,7 @@ impl<'a> PageIndexEvaluator<'a> { } fn apply_predicate_to_column_index( + field_id: i32, predicate: F, field_type: &PrimitiveType, column_index: &ColumnIndexMetaData, @@ -339,46 +341,96 @@ impl<'a> PageIndexEvaluator<'a> { ) }) .collect(), - ColumnIndexMetaData::BYTE_ARRAY(idx) => idx - .min_values_iter() - .zip(idx.max_values_iter()) - .enumerate() - .zip(row_counts.iter()) - .map(|((i, (min, max)), &row_count)| { - predicate( - min.map(|val| { - Datum::new( - field_type.clone(), - PrimitiveLiteral::String(String::from_utf8(val.to_vec()).unwrap()), - ) - }), - max.map(|val| { - Datum::new( - field_type.clone(), - PrimitiveLiteral::String(String::from_utf8(val.to_vec()).unwrap()), - ) - }), + ColumnIndexMetaData::BYTE_ARRAY(idx) => { + // Parquet stores Iceberg string and binary bounds as BYTE_ARRAY. + // Any other field type on a BYTE_ARRAY column (e.g. a non-spec + // BYTE_ARRAY decimal) can't be decoded safely, so skip page + // pruning for it rather than risk pruning pages that match. + if !matches!(field_type, PrimitiveType::String | PrimitiveType::Binary) { + tracing::debug!( + field_id, + %field_type, + "Skipping page-index pruning: BYTE_ARRAY column index on a non-string/binary field" + ); + return Ok(None); + } + + let mut page_filter = Vec::with_capacity(row_counts.len()); + for ((i, (min, max)), &row_count) in idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() + .zip(row_counts.iter()) + { + // A bound that won't decode (e.g. a min/max stat truncated + // mid-UTF-8-sequence) means this column's page index can't + // be trusted, so skip pruning for the whole column rather + // than abort the scan. + let (min, max) = match ( + min.map(|val| Self::byte_array_bound_to_datum(field_type, val)) + .transpose(), + max.map(|val| Self::byte_array_bound_to_datum(field_type, val)) + .transpose(), + ) { + (Ok(min), Ok(max)) => (min, max), + (Err(err), _) | (_, Err(err)) => { + tracing::debug!( + field_id, + %field_type, + page_index = i, + %err, + "Skipping page-index pruning: undecodable BYTE_ARRAY page bound" + ); + return Ok(None); + } + }; + + page_filter.push(predicate( + min, + max, PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), - ) - }) - .collect(), - ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(_) => { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "unsupported 'FIXED_LEN_BYTE_ARRAY' index type in column_index", - )); + )?); + } + + Ok(page_filter) } - ColumnIndexMetaData::INT96(_) => { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "unsupported 'INT96' index type in column_index", - )); + // Column index types we can't interpret: skip page pruning rather + // than abort the scan. Row-group filtering and the Arrow row filter + // still apply the predicate. + ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(_) | ColumnIndexMetaData::INT96(_) => { + tracing::debug!( + field_id, + %field_type, + "Skipping page-index pruning: unsupported FIXED_LEN_BYTE_ARRAY or INT96 column index" + ); + return Ok(None); } }; Ok(Some(result?)) } + /// Converts a `BYTE_ARRAY` page bound into a [`Datum`] according to the + /// field's primitive type. Parquet stores Iceberg `string` and `binary` + /// bounds as `BYTE_ARRAY`. + fn byte_array_bound_to_datum(field_type: &PrimitiveType, bytes: &[u8]) -> Result { + match field_type { + PrimitiveType::String => { + let value = std::str::from_utf8(bytes).map_err(|err| { + Error::new(ErrorKind::DataInvalid, "Invalid UTF-8 in string page bound") + .with_source(err) + })?; + Ok(Datum::string(value)) + } + PrimitiveType::Binary => Ok(Datum::binary(bytes.to_vec())), + // Defensive: production callers only invoke this for String/Binary. + _ => Err(Error::new( + ErrorKind::Unexpected, + format!("Unsupported primitive type for BYTE_ARRAY page bound: {field_type}"), + )), + } + } + fn visit_inequality( &mut self, reference: &BoundReference, @@ -779,7 +831,9 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use arrow_array::{ArrayRef, Float32Array, RecordBatch, StringArray}; + use arrow_array::{ + ArrayRef, FixedSizeBinaryArray, Float32Array, LargeBinaryArray, RecordBatch, StringArray, + }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::{ @@ -896,6 +950,92 @@ mod tests { Ok((metadata, temp_file)) } + /// Creates a single-column parquet file whose `col_binary` page bounds hold + /// non-UTF-8 bytes, backing the binary path through `BYTE_ARRAY` column + /// indexes. Page 0 is `[0x01]`, page 1 is `[0xff, 0x00]`. + fn create_binary_parquet_file() -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "col_binary", + DataType::LargeBinary, + true, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + // Each page holds 1024 rows of a single value, one page per byte pattern. + let pages: [&[u8]; 2] = [&[0x01], &[0xff, 0x00]]; + + for value in pages { + let array = Arc::new(LargeBinaryArray::from_iter_values(std::iter::repeat_n( + value, 1024, + ))) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + + /// Creates a single-column parquet file with a `FixedSizeBinary(2)` column, + /// producing `FIXED_LEN_BYTE_ARRAY` page bounds. Page 0 is `[0x01, 0x01]`, + /// page 1 is `[0x02, 0x02]`. + fn create_fixed_len_byte_array_parquet_file() -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "col_fixed", + DataType::FixedSizeBinary(2), + true, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + // Each page holds 1024 rows of a single value, one page per byte pattern. + let pages: [&[u8]; 2] = [&[0x01, 0x01], &[0x02, 0x02]]; + + for value in pages { + let array = Arc::new( + FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n(value, 1024)).unwrap(), + ) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + /// Get the test metadata components for testing fn get_test_metadata( metadata: &ParquetMetaData, @@ -1185,6 +1325,176 @@ mod tests { Ok(()) } + #[test] + fn eval_inequality_prunes_binary_pages_with_non_utf8_bounds() -> Result<()> { + let (metadata, _temp_file) = create_binary_parquet_file()?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_binary", + Type::Primitive(PrimitiveType::Binary), + true, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + // Page 0 bounds are [0x01], page 1 bounds are [0xff, 0x00]; only page 1 + // exceeds [0x80]. Decoding [0xff, 0x00] as UTF-8 would panic. + let filter = Reference::new("col_binary") + .greater_than(Datum::binary(vec![0x80u8])) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + let expected = vec![RowSelector::skip(1024), RowSelector::select(1024)]; + + assert_eq!(result, expected); + + Ok(()) + } + + #[test] + fn eval_skips_pruning_for_byte_array_decimal_bounds() -> Result<()> { + // A non-spec writer can store a decimal column as BYTE_ARRAY. Those + // bounds can't be decoded safely (Parquet may truncate them, and + // truncation only preserves lexicographic byte order, not decimal + // order), so the evaluator skips page pruning rather than prune pages + // that might match. + let (metadata, _temp_file) = create_binary_parquet_file()?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_decimal", + Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }), + true, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + // A predicate that would prune every page if the bounds were decoded. + let filter = Reference::new("col_decimal") + .greater_than(Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(99999, 2), + 10, + )?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + // Both 1024-row pages survive: no page pruning is applied. + assert_eq!(result, vec![RowSelector::select(2048)]); + + Ok(()) + } + + #[test] + fn eval_skips_pruning_for_fixed_len_byte_array() -> Result<()> { + // FIXED_LEN_BYTE_ARRAY page indexes back spec-conforming fixed, uuid, + // and decimal(P > 18) columns. The evaluator can't interpret those + // bounds, so it skips page pruning rather than abort the scan. + let (metadata, _temp_file) = create_fixed_len_byte_array_parquet_file()?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_fixed", + Type::Primitive(PrimitiveType::Fixed(2)), + true, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + // A predicate that would prune every page if the bounds were decoded. + let filter = Reference::new("col_fixed") + .greater_than(Datum::fixed(vec![0xffu8, 0xff])) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + // All rows survive: no page pruning is applied for FIXED_LEN_BYTE_ARRAY. + assert_eq!(result, vec![RowSelector::select(2048)]); + + Ok(()) + } + + #[test] + fn eval_skips_pruning_for_non_utf8_string_bound() -> Result<()> { + // A writer can truncate a string min/max stat mid-UTF-8-sequence, + // leaving a BYTE_ARRAY page bound that isn't valid UTF-8. Decoding it + // as a string fails, so the evaluator skips page pruning for the column + // rather than aborting the whole file read. + let (metadata, _temp_file) = create_binary_parquet_file()?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_string", + Type::Primitive(PrimitiveType::String), + true, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + // Page 1's bound [0xff, 0x00] isn't valid UTF-8. A predicate that would + // otherwise prune every page must not: the undecodable bound forces a + // skip rather than an error. + let filter = Reference::new("col_string") + .greater_than(Datum::string("zzz")) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + // Both pages survive: no page pruning is applied when a bound won't decode. + assert_eq!(result, vec![RowSelector::select(2048)]); + + Ok(()) + } + #[test] fn eval_starts_with_error_float_col() -> Result<()> { let (metadata, _temp_file) = create_test_parquet_file()?; From f64575819ec5d61a1d8ffd7740831a2cb64df5cd Mon Sep 17 00:00:00 2001 From: Andrei Tserakhau Date: Sun, 27 Sep 2026 02:03:28 +0000 Subject: [PATCH 16/63] feat(writer): add PositionDeleteInput builder (#3230) * feat(writer): add PositionDeleteInput builder Add PositionDeleteInputBuilder, a small helper that assembles a spec-conforming position delete RecordBatch from (path, pos) rows. The batch has exactly the two required position delete columns (file_path: Utf8, pos: Int64), both non-null and carrying the reserved field ids, so it passes PositionDeleteFileWriter's per-batch validation and can be handed straight to that writer. An empty builder still produces a valid 0-row batch. To keep a single source of truth for the field-id wiring, the Arrow schema is no longer hand-wired in the builder. This commit un-gates the previously test-only POSITION_DELETE_ARROW_SCHEMA static and position_delete_arrow_schema() helper in position_delete_writer.rs, making the helper pub(crate), and the builder now obtains the shared Arrow projection through it as a cheap Arc clone. extend accepts any iterator of (Into, i64) rows, and build debug-asserts the paths/positions length invariant. The module docs note that rows are materialized into two heap Vecs before the batch is built (so very large inputs should use one builder per write batch), that the optional third row column is not supported yet, and that position delete files are a v2 construct which v3 forbids, so a format-version gate must be applied at the transaction/commit layer. Relates to #340 and #2218. * refactor(writer): replace PositionDeleteInput builder with a sorted position_delete_batch fn Per review, collapse the PositionDeleteInputBuilder surface (new/with_capacity/push/extend/len/is_empty/build) into a single free function, position_delete_batch(rows). A FooBuilder is expected to build a Foo, but the builder's build() only returned a bare RecordBatch, so a plain function reads more clearly and keeps the API as tight as possible. The function collects rows into a BTreeMap keyed by file path. This deduplicates repeated (file_path, pos) pairs and, when iterated, yields the spec-required ascending (file_path, pos) order for free, so callers no longer have to pre-sort their input. A negative pos is rejected with DataInvalid. The batch is still built from the writer's shared crate internal Arrow schema, and empty input yields a valid 0-row batch. * refactor(writer): model position deletes as a crate-internal PositionDeletes accumulator Following blackmwk's review of the redesigned position delete assembly, replace the free position_delete_batch function with a small crate-internal PositionDeletes accumulator. Callers build up deletes with insert(path, pos) and render them once via to_record_batch(). Positions are now taken as u64: a row position is non-negative by construction, so the old negative-i64 rejection is gone. The batch's Int64 pos column is filled with a checked u64-to-i64 conversion that returns DataInvalid rather than panicking on a value past i64::MAX. Rows stay keyed by a BTreeMap, so iteration yields the spec-required (file_path, pos) order with duplicate pairs removed, and the two columns are now built with Arrow StringBuilder and Int64Builder instead of intermediate Vecs. The batch is assembled against the writer's shared position_delete_arrow_schema, and an empty accumulator still yields a valid 0-row batch. Nothing outside the crate consumes this yet, so the whole surface is pub(crate) and the module carries a dead-code allowance until the write-path plumbing lands. The public API drops the position_delete_batch entry accordingly. * refactor(writer): stream PositionDeletes as fixed-size record batches Follow-up to blackmwk's review on #3230. - Add PositionDeletes::to_record_batches(max_rows), a lazy iterator that builds each fixed-size batch on demand instead of materializing the whole delete set at once, bounding peak memory over large delete sets. It walks the paths and positions in sorted (file_path, pos) order, splitting on max_rows boundaries; a path whose positions overflow one batch resumes in the next. A max_rows of 0 yields a single DataInvalid error rather than panicking, and the stream stops after any error. - Build the file_path column with StringBuilder::append_value_n, appending each path's run within a batch in one call rather than once per row. - Factor a shared private build_batch helper so to_record_batch (single batch) and to_record_batches share the column-building logic. - Document that duplicate (file_path, pos) inserts collapse silently, which is intentional for position deletes. * refactor(writer): debug_assert against duplicate and oversized position deletes --- crates/iceberg/src/writer/base_writer/mod.rs | 1 + .../base_writer/position_delete_input.rs | 673 ++++++++++++++++++ .../base_writer/position_delete_writer.rs | 12 +- 3 files changed, 681 insertions(+), 5 deletions(-) create mode 100644 crates/iceberg/src/writer/base_writer/position_delete_input.rs diff --git a/crates/iceberg/src/writer/base_writer/mod.rs b/crates/iceberg/src/writer/base_writer/mod.rs index 5a23931f25..005260dcf1 100644 --- a/crates/iceberg/src/writer/base_writer/mod.rs +++ b/crates/iceberg/src/writer/base_writer/mod.rs @@ -19,4 +19,5 @@ pub mod data_file_writer; pub mod equality_delete_writer; +pub(crate) mod position_delete_input; pub mod position_delete_writer; diff --git a/crates/iceberg/src/writer/base_writer/position_delete_input.rs b/crates/iceberg/src/writer/base_writer/position_delete_input.rs new file mode 100644 index 0000000000..483739f685 --- /dev/null +++ b/crates/iceberg/src/writer/base_writer/position_delete_input.rs @@ -0,0 +1,673 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Accumulates position delete rows and renders them as spec-conforming [`RecordBatch`]es. +//! +//! [`PositionDeletes`] collects `(file_path, pos)` pairs and emits the two required position +//! delete columns — `file_path` (`Utf8`) and `pos` (`Int64`) — sorted by `(file_path, pos)` +//! with duplicate pairs removed, as the spec requires. Rows can be drawn either as one batch +//! ([`to_record_batch`](PositionDeletes::to_record_batch)) or, to bound peak memory over a +//! large delete set, lazily as fixed-size batches +//! ([`to_record_batches`](PositionDeletes::to_record_batches)). Either batch is exactly what +//! [`PositionDeleteFileWriter`](super::position_delete_writer::PositionDeleteFileWriter) +//! accepts, so it can be handed straight to that writer. +//! +//! Position delete files are a v2 construct. v3 tables use deletion vectors and forbid adding +//! new position delete files, so a format-version gate must be applied at the +//! transaction/commit layer before routing v3 writes here. + +// Nothing wires `PositionDeletes` into a writer yet; the write-path plumbing lands in a follow-up. +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::sync::Arc; + +use arrow_array::builder::{Int64Builder, StringBuilder}; +use arrow_array::{ArrayRef, RecordBatch}; +use roaring::RoaringTreemap; + +use crate::{Error, ErrorKind, Result}; + +/// Accumulates `(file_path, pos)` position delete rows. +/// +/// Keying paths in a [`BTreeMap`] and positions in a [`RoaringTreemap`] deduplicates repeated +/// `(file_path, pos)` pairs and, when iterated, yields the spec-required ascending +/// `(file_path, pos)` order for free, so callers may [`insert`](Self::insert) rows in any +/// order. +#[derive(Debug, Default)] +pub(crate) struct PositionDeletes { + rows: BTreeMap, +} + +impl PositionDeletes { + /// Creates an empty accumulator. + pub(crate) fn new() -> Self { + Self::default() + } + + /// Records that row `pos` of `path` is deleted. `pos` is a row position, so it is + /// non-negative by construction. + /// + /// A duplicate `(file_path, pos)` or a position exceeding `i64::MAX` is a caller bug, + /// caught by [`debug_assert!`] in debug builds so it surfaces early in debug/CI (a + /// position delete file must not list the same `(file_path, pos)` twice, and the `pos` + /// column is `Int64`). In release those assertions compile out and the old behavior + /// stands: the backing [`RoaringTreemap`] silently collapses a duplicate pair, and an + /// oversized position is still rejected as [`ErrorKind::DataInvalid`] when the batch is + /// built. + pub(crate) fn insert(&mut self, path: impl Into, pos: u64) { + debug_assert!(pos <= i64::MAX as u64, "position {pos} exceeds i64::MAX"); + let inserted = self.rows.entry(path.into()).or_default().insert(pos); + debug_assert!( + inserted, + "duplicate position delete: the same (file_path, pos) was inserted twice" + ); + } + + /// Returns whether no positions have been recorded. `insert` never leaves an empty + /// treemap behind, so an empty map means no positions. + pub(crate) fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + /// Returns the total number of recorded positions across all paths. + pub(crate) fn len(&self) -> usize { + self.rows + .values() + .map(|positions| positions.len()) + .sum::() as usize + } + + /// Renders all accumulated rows as one spec-conforming position delete [`RecordBatch`]. + /// + /// The batch has exactly the two required position delete columns, built against the + /// writer's shared [`position_delete_arrow_schema`](super::position_delete_writer::position_delete_arrow_schema) + /// so the reserved field ids stay wired in. Rows are emitted sorted by `(file_path, pos)` + /// with duplicate pairs removed. An empty accumulator yields a valid 0-row batch. + /// + /// This materializes every row at once; prefer [`to_record_batches`](Self::to_record_batches) + /// to bound peak memory over a large delete set. + pub(crate) fn to_record_batch(&self) -> Result { + // `usize::MAX` never caps a real position count, so this drains everything into one + // batch (a 0-row batch when empty). + Self::build_batch(&mut self.stream().peekable(), usize::MAX) + } + + /// Lazily renders the accumulated rows as position delete [`RecordBatch`]es of at most + /// `max_rows` rows each, so the whole delete set is never materialized at once. + /// + /// Batches carry the two required columns in ascending `(file_path, pos)` order with + /// duplicate pairs removed, split on `max_rows` boundaries; a single path whose positions + /// overflow one batch simply resumes in the next. Each batch is built on demand as the + /// iterator is polled — nothing is pre-collected — and uses the writer's shared + /// [`position_delete_arrow_schema`](super::position_delete_writer::position_delete_arrow_schema). + /// An empty accumulator yields no batches. + /// + /// `max_rows` must be greater than zero. Rather than panic, a `max_rows` of `0` makes the + /// iterator yield a single [`ErrorKind::DataInvalid`] error and then stop. + pub(crate) fn to_record_batches( + &self, + max_rows: usize, + ) -> impl Iterator> + '_ { + let mut rows = self.stream().peekable(); + let mut done = false; + std::iter::from_fn(move || { + if done { + return None; + } + if max_rows == 0 { + // Invalid, but never panic: surface it once and stop the iterator. + done = true; + return Some(Err(Error::new( + ErrorKind::DataInvalid, + "max_rows must be greater than 0", + ))); + } + // Nothing left to emit -> end the iteration. + rows.peek()?; + let batch = Self::build_batch(&mut rows, max_rows); + // A failed batch (e.g. a position overflowing `i64`) leaves its offending row + // unconsumed, so stop after the first error rather than retry it forever. + if batch.is_err() { + done = true; + } + Some(batch) + }) + } + + /// Flattens the accumulator into its rows in ascending `(file_path, pos)` order. + /// + /// The `BTreeMap` iterates paths ascending and each `RoaringTreemap` iterates positions + /// ascending, so the stream is sorted by `(file_path, pos)` and already deduplicated. + fn stream(&self) -> impl Iterator + '_ { + self.rows + .iter() + .flat_map(|(path, positions)| positions.iter().map(move |pos| (path.as_str(), pos))) + } + + /// Builds one position delete [`RecordBatch`] of at most `max_rows` rows drained from + /// `rows`, which must yield rows already in `(file_path, pos)` order. + /// + /// Consecutive rows sharing a path form a run appended to the `file_path` column in a + /// single [`StringBuilder::append_value_n`] call, keeping that column run-length friendly + /// rather than repeating the path per row. The batch is built against the writer's shared + /// Arrow projection (a cheap `Arc` clone) so the reserved field ids stay wired in. An + /// exhausted `rows` yields a valid 0-row batch. + fn build_batch<'a, I>( + rows: &mut std::iter::Peekable, + max_rows: usize, + ) -> Result + where + I: Iterator, + { + let mut path_builder = StringBuilder::new(); + let mut pos_builder = Int64Builder::new(); + + let mut taken = 0usize; + while taken < max_rows { + let Some(&(path, _)) = rows.peek() else { + break; + }; + // Consume this path's run: contiguous rows sharing `path`, up to the batch's cap. + let mut run = 0usize; + while taken < max_rows { + let Some(&(next_path, pos)) = rows.peek() else { + break; + }; + if next_path != path { + break; + } + // The `pos` column is `Int64`; positions are `u64`, so guard the top of the + // range with a checked conversion rather than a silent wrap. + let pos = i64::try_from(pos) + .map_err(|_| Error::new(ErrorKind::DataInvalid, "position exceeds i64::MAX"))?; + pos_builder.append_value(pos); + rows.next(); + run += 1; + taken += 1; + } + // One append per run of `run` rows, not one per row. + path_builder.append_value_n(path, run); + } + + let schema = super::position_delete_writer::position_delete_arrow_schema(); + RecordBatch::try_new(schema, vec![ + Arc::new(path_builder.finish()) as ArrayRef, + Arc::new(pos_builder.finish()) as ArrayRef, + ]) + .map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Failed to build position delete record batch: {e}"), + ) + }) + } +} + +#[cfg(test)] +mod test { + use arrow_array::{Array, Int64Array, StringArray}; + use arrow_schema::DataType; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::file::properties::WriterProperties; + use tempfile::TempDir; + + use super::*; + use crate::io::FileIO; + use crate::metadata_columns::{ + RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS, + RESERVED_FIELD_ID_DELETE_FILE_PATH, RESERVED_FIELD_ID_DELETE_FILE_POS, + }; + use crate::spec::{DataContentType, DataFileFormat}; + use crate::writer::base_writer::position_delete_writer::{ + PositionDeleteFileWriterBuilder, position_delete_schema, + }; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + use crate::writer::{IcebergWriter, IcebergWriterBuilder}; + + /// Asserts a batch has the exact two-column position delete schema: names, Arrow types, + /// non-null flags, and the reserved field ids in the Parquet field-id metadata. + fn assert_position_delete_schema(batch: &RecordBatch) { + assert_eq!(batch.num_columns(), 2); + let schema = batch.schema(); + + let path = schema.field(0); + assert_eq!(path.name(), RESERVED_COL_NAME_DELETE_FILE_PATH); + assert_eq!(path.data_type(), &DataType::Utf8); + assert!(!path.is_nullable()); + assert_eq!( + path.metadata().get(PARQUET_FIELD_ID_META_KEY).unwrap(), + &RESERVED_FIELD_ID_DELETE_FILE_PATH.to_string() + ); + + let pos = schema.field(1); + assert_eq!(pos.name(), RESERVED_COL_NAME_DELETE_FILE_POS); + assert_eq!(pos.data_type(), &DataType::Int64); + assert!(!pos.is_nullable()); + assert_eq!( + pos.metadata().get(PARQUET_FIELD_ID_META_KEY).unwrap(), + &RESERVED_FIELD_ID_DELETE_FILE_POS.to_string() + ); + } + + /// Extracts the `(file_path, pos)` columns of a batch as parallel vectors. + fn columns(batch: &RecordBatch) -> (Vec, Vec) { + let paths = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let positions = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(paths.null_count(), 0); + assert_eq!(positions.null_count(), 0); + ( + paths.iter().map(|s| s.unwrap().to_string()).collect(), + positions.values().to_vec(), + ) + } + + #[test] + fn test_column_shape_and_field_ids() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 1); + let batch = deletes.to_record_batch().unwrap(); + assert_position_delete_schema(&batch); + } + + #[test] + fn test_unsorted_inserts_sorted_by_path_then_pos() { + let mut deletes = PositionDeletes::new(); + // Inserted out of order, across multiple paths; both the paths and the positions + // within a path must come out sorted. + deletes.insert("s3://bucket/data/f1.parquet", 2); + deletes.insert("s3://bucket/data/f0.parquet", 4); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + let batch = deletes.to_record_batch().unwrap(); + let (paths, positions) = columns(&batch); + assert_eq!(paths, vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f1.parquet", + ]); + assert_eq!(positions, vec![1, 4, 2]); + } + + // Release behavior: a duplicate `(file_path, pos)` is silently deduped. In debug builds + // that same input trips a `debug_assert!` (see `insert_duplicate_pair_debug_asserts`), so + // this test is release-only. + #[cfg(not(debug_assertions))] + #[test] + fn test_duplicate_pairs_deduped() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + assert_eq!(deletes.len(), 2); + let batch = deletes.to_record_batch().unwrap(); + let (paths, positions) = columns(&batch); + assert_eq!(batch.num_rows(), 2); + assert_eq!(paths, vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + ]); + assert_eq!(positions, vec![1, 3]); + } + + // Debug behavior: re-inserting the same `(file_path, pos)` is a caller bug that trips a + // `debug_assert!` rather than silently deduping (the release behavior asserted by + // `test_duplicate_pairs_deduped`). + #[cfg(debug_assertions)] + #[test] + #[should_panic(expected = "duplicate position delete")] + fn insert_duplicate_pair_debug_asserts() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 3); + } + + #[test] + fn test_len_and_is_empty() { + let mut deletes = PositionDeletes::new(); + assert!(deletes.is_empty()); + assert_eq!(deletes.len(), 0); + + deletes.insert("s3://bucket/data/f0.parquet", 1); + deletes.insert("s3://bucket/data/f0.parquet", 2); + deletes.insert("s3://bucket/data/f1.parquet", 9); + assert!(!deletes.is_empty()); + assert_eq!(deletes.len(), 3); + + // A duplicate pair does not change the count. This is release behavior; in debug the + // duplicate insert trips a `debug_assert!`, so the check is release-only here. + #[cfg(not(debug_assertions))] + { + deletes.insert("s3://bucket/data/f0.parquet", 1); + assert_eq!(deletes.len(), 3); + } + } + + #[test] + fn test_empty_builds_valid_zero_row_batch() { + let deletes = PositionDeletes::new(); + let batch = deletes.to_record_batch().unwrap(); + assert_eq!(batch.num_rows(), 0); + // Even with no rows the schema must be the full position delete schema. + assert_position_delete_schema(&batch); + } + + // Release behavior: a position beyond `i64::MAX` cannot fit the `Int64` column and is + // rejected as `DataInvalid` when the batch is built (the release safety net). In debug + // builds that same input trips a `debug_assert!` in `insert` up front (see + // `insert_position_above_i64_max_debug_asserts`), so this test is release-only. + #[cfg(not(debug_assertions))] + #[test] + fn test_position_above_i64_max_rejected() { + let mut deletes = PositionDeletes::new(); + // A position beyond `i64::MAX` cannot fit the `Int64` column and must be rejected + // rather than silently wrapped. + deletes.insert("s3://bucket/data/f0.parquet", i64::MAX as u64 + 1); + let err = deletes.to_record_batch().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + // Debug behavior: a position beyond `i64::MAX` is a caller bug that trips a + // `debug_assert!` in `insert` up front, rather than being rejected only when the batch is + // built (the release behavior asserted by `test_position_above_i64_max_rejected`). + #[cfg(debug_assertions)] + #[test] + #[should_panic(expected = "exceeds i64::MAX")] + fn insert_position_above_i64_max_debug_asserts() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", i64::MAX as u64 + 1); + } + + /// Drains [`PositionDeletes::to_record_batches`] into per-batch `(paths, positions)` + /// column pairs, unwrapping each batch. + fn batched_columns(deletes: &PositionDeletes, max_rows: usize) -> Vec<(Vec, Vec)> { + deletes + .to_record_batches(max_rows) + .map(|batch| columns(&batch.unwrap())) + .collect() + } + + /// Concatenates per-batch column pairs into one flattened `(paths, positions)` pair. + fn flatten(batches: &[(Vec, Vec)]) -> (Vec, Vec) { + let mut paths = Vec::new(); + let mut positions = Vec::new(); + for (batch_paths, batch_positions) in batches { + paths.extend(batch_paths.iter().cloned()); + positions.extend(batch_positions.iter().copied()); + } + (paths, positions) + } + + #[test] + fn test_to_record_batches_splits_and_preserves_order() { + let mut deletes = PositionDeletes::new(); + // Unsorted inserts across three paths; 6 rows total. + for (path, pos) in [ + ("s3://bucket/data/f1.parquet", 5), + ("s3://bucket/data/f0.parquet", 7), + ("s3://bucket/data/f0.parquet", 1), + ("s3://bucket/data/f2.parquet", 9), + ("s3://bucket/data/f1.parquet", 2), + ("s3://bucket/data/f0.parquet", 4), + ] { + deletes.insert(path, pos); + } + + // 6 rows at 2 per batch -> exactly 3 full batches. + let batches = batched_columns(&deletes, 2); + assert_eq!(batches.len(), 3); + for (paths, positions) in &batches { + assert_eq!(paths.len(), 2); + assert_eq!(positions.len(), 2); + } + // Every streamed batch carries the full position delete schema. + for batch in deletes.to_record_batches(2) { + assert_position_delete_schema(&batch.unwrap()); + } + + // Flattened, the stream is the same rows in the same sorted order as the single batch. + let (flat_paths, flat_positions) = flatten(&batches); + let (whole_paths, whole_positions) = columns(&deletes.to_record_batch().unwrap()); + assert_eq!(flat_paths, whole_paths); + assert_eq!(flat_positions, whole_positions); + assert_eq!(whole_paths, vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f1.parquet", + "s3://bucket/data/f1.parquet", + "s3://bucket/data/f2.parquet", + ]); + assert_eq!(whole_positions, vec![1, 4, 7, 2, 5, 9]); + } + + #[test] + fn test_to_record_batches_uneven_last_batch() { + let mut deletes = PositionDeletes::new(); + for pos in 0..5 { + deletes.insert("s3://bucket/data/f0.parquet", pos); + } + + // 5 rows at 2 per batch -> two full batches and a 1-row remainder. + let sizes: Vec = deletes + .to_record_batches(2) + .map(|batch| batch.unwrap().num_rows()) + .collect(); + assert_eq!(sizes, vec![2, 2, 1]); + } + + #[test] + fn test_to_record_batches_path_spans_chunk_boundary() { + let mut deletes = PositionDeletes::new(); + // A single path with more positions than fit one batch: its run must resume across + // batch boundaries and stay correct. + for pos in 1..=5 { + deletes.insert("s3://bucket/data/f0.parquet", pos); + } + + let batches = batched_columns(&deletes, 2); + assert_eq!(batches.len(), 3); + // Same path in every batch, positions still ascending and contiguous across the split. + for (paths, _) in &batches { + assert!(paths.iter().all(|p| p == "s3://bucket/data/f0.parquet")); + } + let (paths, positions) = flatten(&batches); + assert_eq!(paths.len(), 5); + assert_eq!(positions, vec![1, 2, 3, 4, 5]); + } + + #[test] + fn test_to_record_batches_zero_max_rows_yields_error_without_panic() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + let mut it = deletes.to_record_batches(0); + // A `max_rows` of 0 is invalid, but must surface as an error rather than a panic... + let err = it.next().expect("expected one error item").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + // ...exactly once, after which the iterator is exhausted. + assert!(it.next().is_none()); + } + + // Release-only: inserting a position beyond `i64::MAX` trips a `debug_assert!` in debug + // builds, so the streaming overflow safety net can only be exercised in release. + #[cfg(not(debug_assertions))] + #[test] + fn test_to_record_batches_stops_after_overflow_error() { + let mut deletes = PositionDeletes::new(); + // A position beyond `i64::MAX` cannot fit the `Int64` column; the streaming path must + // surface the error and then stop rather than retry the unconsumed row forever. + deletes.insert("s3://bucket/data/f0.parquet", i64::MAX as u64 + 1); + + let mut it = deletes.to_record_batches(2); + let err = it.next().expect("expected one error item").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(it.next().is_none()); + } + + #[test] + fn test_to_record_batches_empty_yields_no_batches() { + let deletes = PositionDeletes::new(); + // Empty accumulator streams nothing... + assert_eq!(deletes.to_record_batches(4).count(), 0); + // ...while the single-batch path still returns a valid 0-row batch. + let batch = deletes.to_record_batch().unwrap(); + assert_eq!(batch.num_rows(), 0); + assert_position_delete_schema(&batch); + } + + // Release-only: inserting the same `(file_path, pos)` twice trips a `debug_assert!` in + // debug builds, so the streaming dedup behavior can only be exercised in release. + #[cfg(not(debug_assertions))] + #[test] + fn test_to_record_batches_dedups_repeated_pairs() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + // The duplicate collapses, so a generous batch holds just the two distinct rows. + let batches = batched_columns(&deletes, 100); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].0, vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + ]); + assert_eq!(batches[0].1, vec![1, 3]); + } + + #[test] + fn test_to_record_batches_repeats_path_across_run() { + let mut deletes = PositionDeletes::new(); + // A run of one path followed by a second path, all within a single batch: the + // `append_value_n` run must repeat the file_path value once per position. + for pos in [1, 2, 3, 4] { + deletes.insert("s3://bucket/data/f0.parquet", pos); + } + deletes.insert("s3://bucket/data/f1.parquet", 8); + + let batches = batched_columns(&deletes, 100); + assert_eq!(batches.len(), 1); + let (paths, positions) = &batches[0]; + assert_eq!(paths, &vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f1.parquet", + ]); + assert_eq!(positions, &vec![1, 2, 3, 4, 8]); + } + + /// Wires a real [`PositionDeleteFileWriter`] over a local filesystem for the integration + /// test: the file IO, location/name generators, and the Parquet-backed rolling writer + /// configured with the [`position_delete_schema`]. + fn writer_setup( + temp_dir: &TempDir, + ) -> PositionDeleteFileWriterBuilder< + ParquetWriterBuilder, + DefaultLocationGenerator, + DefaultFileNameGenerator, + > { + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + let parquet_writer_builder = ParquetWriterBuilder::new( + WriterProperties::builder().build(), + position_delete_schema(), + ); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io, + location_gen, + file_name_gen, + ); + PositionDeleteFileWriterBuilder::new(rolling_writer_builder) + } + + #[tokio::test] + async fn test_batch_feeds_position_delete_writer() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let builder = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + let mut deletes = PositionDeletes::new(); + // Deliberately unsorted; `to_record_batch` sorts to the order the writer needs. + deletes.insert("s3://bucket/data/f1.parquet", 2); + deletes.insert("s3://bucket/data/f0.parquet", 4); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + writer.write(deletes.to_record_batch()?).await?; + let data_files = writer.close().await?; + + assert_eq!(data_files.len(), 1); + let data_file = &data_files[0]; + assert_eq!(data_file.content_type(), DataContentType::PositionDeletes); + assert_eq!(data_file.record_count, deletes.len() as u64); + + Ok(()) + } + + #[tokio::test] + async fn test_streamed_batches_feed_position_delete_writer() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let builder = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + let mut deletes = PositionDeletes::new(); + // Deliberately unsorted, across paths, more rows than the batch cap so more than one + // batch is streamed to the writer. + deletes.insert("s3://bucket/data/f1.parquet", 2); + deletes.insert("s3://bucket/data/f0.parquet", 4); + deletes.insert("s3://bucket/data/f0.parquet", 1); + deletes.insert("s3://bucket/data/f1.parquet", 5); + deletes.insert("s3://bucket/data/f2.parquet", 9); + + // Each fixed-size batch is a valid position delete batch the writer accepts, and the + // batches arrive in the sorted order the writer requires. + let mut streamed = 0usize; + for batch in deletes.to_record_batches(2) { + writer.write(batch?).await?; + streamed += 1; + } + assert_eq!(streamed, 3); + + let data_files = writer.close().await?; + assert_eq!(data_files.len(), 1); + let data_file = &data_files[0]; + assert_eq!(data_file.content_type(), DataContentType::PositionDeletes); + assert_eq!(data_file.record_count, deletes.len() as u64); + + Ok(()) + } +} diff --git a/crates/iceberg/src/writer/base_writer/position_delete_writer.rs b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs index 29b65b74ad..3b023c9c62 100644 --- a/crates/iceberg/src/writer/base_writer/position_delete_writer.rs +++ b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs @@ -65,9 +65,9 @@ static POSITION_DELETE_SCHEMA: Lazy = Lazy::new(|| { }); /// [`POSITION_DELETE_SCHEMA`] converted to Arrow, keeping the reserved field ids in each -/// field's Parquet field-id metadata. Test-only for now: callers configure the writer with -/// the Iceberg [`position_delete_schema`], so no non-test code needs the Arrow form yet. -#[cfg(test)] +/// field's Parquet field-id metadata. This is the crate-internal Arrow projection of a +/// position delete file, shared with +/// [`PositionDeletes`](super::position_delete_input::PositionDeletes). static POSITION_DELETE_ARROW_SCHEMA: Lazy = Lazy::new(|| { Arc::new( crate::arrow::schema_to_arrow_schema(&POSITION_DELETE_SCHEMA) @@ -85,8 +85,10 @@ pub fn position_delete_schema() -> SchemaRef { } /// Returns the canonical Arrow schema of a position delete file. -#[cfg(test)] -fn position_delete_arrow_schema() -> arrow_schema::SchemaRef { +/// +/// A cheap [`Arc`] clone of the shared static, used by +/// [`PositionDeletes`](super::position_delete_input::PositionDeletes). +pub(crate) fn position_delete_arrow_schema() -> arrow_schema::SchemaRef { POSITION_DELETE_ARROW_SCHEMA.clone() } From 86d6618804a902eac10f09c09a565aca5dad0d46 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 28 Sep 2026 17:09:53 +0000 Subject: [PATCH 17/63] ci: auto-label DataFusion issues and PRs (#3278) ci: auto-label issues and PRs that mention DataFusion Contributors without triage permission cannot add labels. Use github/issue-labeler to apply the datafusion label when an issue or PR title or body mentions DataFusion. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/labeler.yml | 21 ++++++++++++++++++ .github/workflows/labeler.yml | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 .github/labeler.yml create mode 100644 .github/workflows/labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000000..0cf48f105a --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Issue and pull request labeler configuration: https://github.com/github/issue-labeler +# Labels issues and PRs whose title or body matches the regex. +datafusion: + - '/datafusion/i' diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000000..016800e0dc --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Issue and Pull Request Labeler" +on: + issues: + types: [opened, edited] + pull_request_target: # zizmor: ignore[dangerous-triggers] + types: [opened, edited] + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + triage: + if: github.repository_owner == 'apache' + runs-on: ubuntu-slim + steps: + - uses: github/issue-labeler@ccca1e16fcb66762200e616fc29b57b940c1cb67 # v3.5 + with: + configuration-path: .github/labeler.yml # label -> regex rules + enable-versioned-regex: 0 # required; 0 = no versioned configs + include-title: 1 # match title + include-body: 1 # match body (default) From 5a91091c53a5dfebcc8f85f78f0eeedb269d0b87 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Mon, 28 Sep 2026 18:08:55 +0000 Subject: [PATCH 18/63] chore: Extending collaborator list to help with review capacity (#3289) Co-authored-by: ovoievodin --- .asf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.asf.yaml b/.asf.yaml index 974c6b512d..e0b840b259 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -93,6 +93,7 @@ github: - c-thiel - mbutrovich - CTTY + - comphead ghp_branch: gh-pages ghp_path: / copilot_code_review: From 8cb2adeddb4f8da1ca7bd86ca303337f011c12e2 Mon Sep 17 00:00:00 2001 From: Alex Stephen <1325798+rambleraptor@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:41:04 +0000 Subject: [PATCH 19/63] fix(python): raise instead of panicking in the manifest bindings (#3241) There's some assorted places that we need to raise errors instead of panicking. --- bindings/python/src/data_file.rs | 10 ++-- bindings/python/src/manifest.rs | 33 +++++++------ bindings/python/tests/test_manifest.py | 64 ++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/bindings/python/src/data_file.rs b/bindings/python/src/data_file.rs index b0e42e7d73..7051e83280 100644 --- a/bindings/python/src/data_file.rs +++ b/bindings/python/src/data_file.rs @@ -23,6 +23,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyBytes; +use crate::error::to_py_err; + #[pyclass()] pub struct PyPrimitiveLiteral { inner: PrimitiveLiteral, @@ -120,20 +122,20 @@ impl PyDataFile { } #[getter] - fn upper_bounds(&self) -> HashMap> { + fn upper_bounds(&self) -> PyResult>> { self.inner .upper_bounds() .iter() - .map(|(k, v)| (*k, v.to_bytes().unwrap().to_vec())) + .map(|(k, v)| Ok((*k, v.to_bytes().map_err(to_py_err)?.to_vec()))) .collect() } #[getter] - fn lower_bounds(&self) -> HashMap> { + fn lower_bounds(&self) -> PyResult>> { self.inner .lower_bounds() .iter() - .map(|(k, v)| (*k, v.to_bytes().unwrap().to_vec())) + .map(|(k, v)| Ok((*k, v.to_bytes().map_err(to_py_err)?.to_vec()))) .collect() } diff --git a/bindings/python/src/manifest.rs b/bindings/python/src/manifest.rs index 6c042475b6..15b80c3e3e 100644 --- a/bindings/python/src/manifest.rs +++ b/bindings/python/src/manifest.rs @@ -23,6 +23,7 @@ use iceberg::spec::{ use pyo3::prelude::*; use crate::data_file::PyDataFile; +use crate::error::to_py_err; #[pyclass] pub struct PyManifest { @@ -145,14 +146,13 @@ impl PyManifestFile { } #[getter] - fn partitions(&self) -> Vec { - self.inner - .partitions - .clone() - .unwrap() - .iter() - .map(|s| PyFieldSummary { inner: s.clone() }) - .collect() + fn partitions(&self) -> Option> { + self.inner.partitions.as_ref().map(|partitions| { + partitions + .iter() + .map(|s| PyFieldSummary { inner: s.clone() }) + .collect() + }) } #[getter] @@ -195,11 +195,10 @@ impl PyManifestEntry { } #[pyfunction] -pub fn read_manifest_entries(bs: &[u8]) -> PyManifest { - // TODO: Some error handling - PyManifest { - inner: Manifest::parse_avro(bs).unwrap(), - } +pub fn read_manifest_entries(bs: &[u8]) -> PyResult { + Ok(PyManifest { + inner: Manifest::parse_avro(bs).map_err(to_py_err)?, + }) } #[pyclass] @@ -221,10 +220,10 @@ impl PyManifestList { } #[pyfunction] -pub fn read_manifest_list(bs: &[u8]) -> PyManifestList { - PyManifestList { - inner: ManifestList::parse_with_version(bs, FormatVersion::V2).unwrap(), - } +pub fn read_manifest_list(bs: &[u8]) -> PyResult { + Ok(PyManifestList { + inner: ManifestList::parse_with_version(bs, FormatVersion::V2).map_err(to_py_err)?, + }) } pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/bindings/python/tests/test_manifest.py b/bindings/python/tests/test_manifest.py index e024b28086..147b81fccc 100644 --- a/bindings/python/tests/test_manifest.py +++ b/bindings/python/tests/test_manifest.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from pathlib import Path from typing import Any import pytest @@ -26,7 +27,9 @@ FileFormat, ManifestEntry, ManifestEntryStatus, + ManifestFile, _manifests, + write_manifest_list, ) @@ -171,3 +174,64 @@ def test_read_manifest_entry(generated_manifest_entry_file: str) -> None: assert data_file.split_offsets == [4] assert data_file.equality_ids is None assert data_file.sort_order_id == 0 + + +@pytest.mark.parametrize( + "payload", [b"", b"not-an-avro-file", b"Obj\x01truncated-after-the-avro-magic"] +) +def test_read_manifest_entries_raises_on_invalid_avro(payload: bytes) -> None: + from pyiceberg_core import manifest + + with pytest.raises(ValueError): + manifest.read_manifest_entries(payload) + + +@pytest.mark.parametrize( + "payload", [b"", b"not-an-avro-file", b"Obj\x01truncated-after-the-avro-magic"] +) +def test_read_manifest_list_raises_on_invalid_avro(payload: bytes) -> None: + from pyiceberg_core import manifest + + with pytest.raises(ValueError): + manifest.read_manifest_list(payload) + + +def test_manifest_file_without_partition_summaries(tmp_path: Path) -> None: + """Field summaries are optional, so a manifest list without them reads back as None.""" + from pyiceberg_core import manifest + + manifest_list_file = str(tmp_path / "snap.avro") + io = PyArrowFileIO() + + with write_manifest_list( + format_version=2, + output_file=io.new_output(manifest_list_file), + snapshot_id=25, + parent_snapshot_id=None, + sequence_number=1, + avro_compression="null", + ) as writer: + writer.add_manifests( + [ + ManifestFile.from_args( + manifest_path="s3://bucket/metadata/manifest.avro", + manifest_length=1024, + partition_spec_id=0, + added_snapshot_id=25, + sequence_number=1, + min_sequence_number=1, + added_files_count=1, + existing_files_count=0, + deleted_files_count=0, + added_rows_count=10, + existing_rows_count=0, + deleted_rows_count=0, + partitions=None, + ) + ] + ) + + bs = io.new_input(manifest_list_file).open().read() + manifest_file = manifest.read_manifest_list(bs).entries()[0] + + assert manifest_file.partitions is None From deab0692f6df211a396d6fc0ad21811f8b6c8491 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Tue, 29 Sep 2026 11:49:43 +0000 Subject: [PATCH 20/63] refactor(spec): make UnboundPartitionField fields private and carry source-ids (#3267) * Make UnboundPartitionField fields private and carry source-ids The four public fields are replaced by accessors, and the derived builder's visibility is limited to the crate, so an instance can only be built inside the crate or read out of a partition spec JSON. That is what lets the struct guarantee its own invariant: source_ids always holds at least one id. source_ids: Vec replaces source_id: i32. source_id() returns the single id, and an error for a multi-argument field rather than quietly handing back the first one. The serde module reads either spelling -- source-id, or source-ids for a v3 multi-argument transform -- and writes back the one that matches the field. Binding a multi-argument field now fails with a clear error. A bound PartitionField still carries a single source_id, so until that struct is converted there is nowhere to put the extra ids, and failing loudly beats dropping them on the way into TableCreation or TableUpdate::AddSpec. * Reject source-id and source-ids together in UnboundPartitionField PyIceberg already treats the two spellings as mutually exclusive, and no writer emits both, so accepting a matching pair only let the two readers disagree. Also name both spellings in the missing-id error and note why check_transform_compatibility rejects a multi-argument field. Co-authored-by: Isaac * Make UnboundPartitionField buildable and pass it to add_partition_field Address review: the UnboundPartitionField builder and with_field_id are public, since an unbound field is not expected to be validated yet, and UnboundPartitionSpecBuilder::add_partition_field now takes the field rather than a single source id, name and transform. With the builder public an empty source_ids can be built, so serializing one no longer panics and both spec builders reject it with a clear error. Co-authored-by: Isaac * Validate UnboundPartitionField when it is built Address review: the builder's build() now returns Result and rejects an empty source_ids, following FileScanTask. Deserialization goes through the same builder, so the check lives in one place and the spec builders no longer need their own. Co-authored-by: Isaac --------- Co-authored-by: Isaac --- crates/catalog/rest/src/catalog.rs | 11 +- crates/iceberg/public-api.txt | 15 +- .../arrow/record_batch_partition_splitter.rs | 28 +- crates/iceberg/src/catalog/mod.rs | 31 +- .../src/expr/visitors/expression_evaluator.rs | 5 +- .../visitors/inclusive_metrics_evaluator.rs | 5 +- .../src/expr/visitors/inclusive_projection.rs | 60 ++- crates/iceberg/src/partitioning.rs | 88 ++-- crates/iceberg/src/scan/cache.rs | 11 +- crates/iceberg/src/spec/partition.rs | 461 ++++++++++++++---- crates/iceberg/src/spec/snapshot_summary.rs | 10 +- crates/iceberg/src/spec/table_metadata.rs | 108 ++-- .../src/spec/table_metadata_builder.rs | 304 ++++++++---- 13 files changed, 821 insertions(+), 316 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index aa36a074e7..b7d3b3163c 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -3832,13 +3832,14 @@ mod tests { .properties(HashMap::from([("owner".to_string(), "testx".to_string())])) .partition_spec( UnboundPartitionSpec::builder() - .add_partition_fields(vec![ + .add_partition_field( UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) + .name("id") .transform(Transform::Truncate(3)) - .name("id".to_string()) - .build(), - ]) + .build() + .unwrap(), + ) .unwrap() .build(), ) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index e2e78073e9..58c7db7dd6 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2911,10 +2911,13 @@ pub fn iceberg::spec::TableProperties<'properties>::write_target_file_size_bytes impl<'properties> core::fmt::Debug for iceberg::spec::TableProperties<'properties> pub fn iceberg::spec::TableProperties<'properties>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::UnboundPartitionField -pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option -pub iceberg::spec::UnboundPartitionField::name: alloc::string::String -pub iceberg::spec::UnboundPartitionField::source_id: i32 -pub iceberg::spec::UnboundPartitionField::transform: iceberg::spec::Transform +impl iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::field_id(&self) -> core::option::Option +pub fn iceberg::spec::UnboundPartitionField::name(&self) -> &str +pub fn iceberg::spec::UnboundPartitionField::source_id(&self) -> iceberg::Result +pub fn iceberg::spec::UnboundPartitionField::source_ids(&self) -> &[i32] +pub fn iceberg::spec::UnboundPartitionField::transform(&self) -> iceberg::spec::Transform +pub fn iceberg::spec::UnboundPartitionField::with_field_id(self, field_id: i32) -> Self impl core::clone::Clone for iceberg::spec::UnboundPartitionField pub fn iceberg::spec::UnboundPartitionField::clone(&self) -> iceberg::spec::UnboundPartitionField impl core::cmp::Eq for iceberg::spec::UnboundPartitionField @@ -2922,6 +2925,8 @@ impl core::cmp::PartialEq for iceberg::spec::UnboundPartitionField pub fn iceberg::spec::UnboundPartitionField::eq(&self, other: &iceberg::spec::UnboundPartitionField) -> bool impl core::convert::From for iceberg::spec::UnboundPartitionField pub fn iceberg::spec::UnboundPartitionField::from(field: iceberg::spec::PartitionField) -> Self +impl core::convert::From for iceberg::Result +pub fn iceberg::Result::from(field: iceberg::spec::UnboundPartitionField) -> Self impl core::fmt::Debug for iceberg::spec::UnboundPartitionField pub fn iceberg::spec::UnboundPartitionField::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::spec::UnboundPartitionField @@ -2956,7 +2961,7 @@ impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::UnboundPartitionSp pub fn iceberg::spec::UnboundPartitionSpec::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::UnboundPartitionSpecBuilder impl iceberg::spec::UnboundPartitionSpecBuilder -pub fn iceberg::spec::UnboundPartitionSpecBuilder::add_partition_field(self, source_id: i32, target_name: impl alloc::string::ToString, transformation: iceberg::spec::Transform) -> iceberg::Result +pub fn iceberg::spec::UnboundPartitionSpecBuilder::add_partition_field(self, field: iceberg::spec::UnboundPartitionField) -> iceberg::Result pub fn iceberg::spec::UnboundPartitionSpecBuilder::add_partition_fields(self, fields: impl core::iter::traits::collect::IntoIterator) -> iceberg::Result pub fn iceberg::spec::UnboundPartitionSpecBuilder::build(self) -> iceberg::spec::UnboundPartitionSpec pub fn iceberg::spec::UnboundPartitionSpecBuilder::new() -> Self diff --git a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs index 6b3aa73d22..0b679040c6 100644 --- a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs +++ b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs @@ -244,12 +244,14 @@ mod tests { let partition_spec = Arc::new( PartitionSpecBuilder::new(schema.clone()) .with_spec_id(1) - .add_unbound_field(UnboundPartitionField { - source_id: 1, - field_id: None, - name: "id_bucket".to_string(), - transform: Transform::Identity, - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(), @@ -358,12 +360,14 @@ mod tests { let partition_spec = Arc::new( PartitionSpecBuilder::new(schema.clone()) .with_spec_id(1) - .add_unbound_field(UnboundPartitionField { - source_id: 1, - field_id: None, - name: "id_bucket".to_string(), - transform: Transform::Identity, - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(), diff --git a/crates/iceberg/src/catalog/mod.rs b/crates/iceberg/src/catalog/mod.rs index cf7a94d20f..88681040cd 100644 --- a/crates/iceberg/src/catalog/mod.rs +++ b/crates/iceberg/src/catalog/mod.rs @@ -1103,8 +1103,8 @@ mod tests { PartitionStatisticsFile, PrimitiveType, Schema, Snapshot, SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, SqlViewRepresentation, StatisticsFile, Summary, TableMetadata, TableMetadataBuilder, Transform, Type, - UnboundPartitionSpec, ViewFormatVersion, ViewRepresentation, ViewRepresentations, - ViewVersion, + UnboundPartitionField, UnboundPartitionSpec, ViewFormatVersion, ViewRepresentation, + ViewRepresentations, ViewVersion, }; use crate::table::Table; use crate::test_utils::test_runtime; @@ -1659,11 +1659,32 @@ mod tests { "#, TableUpdate::AddSpec { spec: UnboundPartitionSpec::builder() - .add_partition_field(4, "ts_day".to_string(), Transform::Day) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap() - .add_partition_field(1, "id_bucket".to_string(), Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() - .add_partition_field(2, "id_truncate".to_string(), Transform::Truncate(4)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("id_truncate") + .transform(Transform::Truncate(4)) + .build() + .unwrap(), + ) .unwrap() .build(), }, diff --git a/crates/iceberg/src/expr/visitors/expression_evaluator.rs b/crates/iceberg/src/expr/visitors/expression_evaluator.rs index 122f195897..31ce94a99b 100644 --- a/crates/iceberg/src/expr/visitors/expression_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/expression_evaluator.rs @@ -280,11 +280,12 @@ mod tests { .with_spec_id(1) .add_unbound_field( UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("a".to_string()) .field_id(1) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ) .unwrap() .build() diff --git a/crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs index 06c92ab3e8..8ccd1364d9 100644 --- a/crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs @@ -1661,11 +1661,12 @@ mod test { .with_spec_id(1) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("a".to_string()) .field_id(1) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ]) .unwrap() .build() diff --git a/crates/iceberg/src/expr/visitors/inclusive_projection.rs b/crates/iceberg/src/expr/visitors/inclusive_projection.rs index d9544e4c47..88508af844 100644 --- a/crates/iceberg/src/expr/visitors/inclusive_projection.rs +++ b/crates/iceberg/src/expr/visitors/inclusive_projection.rs @@ -304,11 +304,12 @@ mod tests { .with_spec_id(1) .add_unbound_field( UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("a".to_string()) .field_id(1) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ) .unwrap() .build() @@ -339,12 +340,15 @@ mod tests { let partition_spec = PartitionSpec::builder(arc_schema.clone()) .with_spec_id(1) - .add_unbound_fields(vec![UnboundPartitionField { - source_id: 2, - name: "year".to_string(), - field_id: Some(1000), - transform: Transform::Year, - }]) + .add_unbound_fields(vec![ + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("year".to_string()) + .transform(Transform::Year) + .build() + .unwrap(), + ]) .unwrap() .build() .unwrap(); @@ -374,12 +378,15 @@ mod tests { let partition_spec = PartitionSpec::builder(arc_schema.clone()) .with_spec_id(1) - .add_unbound_fields(vec![UnboundPartitionField { - source_id: 2, - name: "month".to_string(), - field_id: Some(1000), - transform: Transform::Month, - }]) + .add_unbound_fields(vec![ + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("month".to_string()) + .transform(Transform::Month) + .build() + .unwrap(), + ]) .unwrap() .build() .unwrap(); @@ -409,12 +416,15 @@ mod tests { let partition_spec = PartitionSpec::builder(arc_schema.clone()) .with_spec_id(1) - .add_unbound_fields(vec![UnboundPartitionField { - source_id: 2, - name: "day".to_string(), - field_id: Some(1000), - transform: Transform::Day, - }]) + .add_unbound_fields(vec![ + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("day".to_string()) + .transform(Transform::Day) + .build() + .unwrap(), + ]) .unwrap() .build() .unwrap(); @@ -446,11 +456,12 @@ mod tests { .with_spec_id(1) .add_unbound_field( UnboundPartitionField::builder() - .source_id(3) + .source_ids(vec![3]) .name("name_truncate".to_string()) .field_id(3) .transform(Transform::Truncate(4)) - .build(), + .build() + .unwrap(), ) .unwrap() .build() @@ -486,11 +497,12 @@ mod tests { .with_spec_id(1) .add_unbound_field( UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("a_bucket[7]".to_string()) .field_id(1) .transform(Transform::Bucket(7)) - .build(), + .build() + .unwrap(), ) .unwrap() .build() diff --git a/crates/iceberg/src/partitioning.rs b/crates/iceberg/src/partitioning.rs index ef230ef9e5..6ac62c68c4 100644 --- a/crates/iceberg/src/partitioning.rs +++ b/crates/iceberg/src/partitioning.rs @@ -177,7 +177,9 @@ mod tests { use std::sync::Arc; use super::*; - use crate::spec::{NestedField, PrimitiveType, Transform, Type, UnboundPartitionSpec}; + use crate::spec::{ + NestedField, PrimitiveType, Transform, Type, UnboundPartitionField, UnboundPartitionSpec, + }; fn test_schema() -> Schema { Schema::builder() @@ -199,7 +201,14 @@ mod tests { let mut builder = UnboundPartitionSpec::builder().with_spec_id(spec_id); for (source_id, name, transform) in fields { builder = builder - .add_partition_field(source_id, name, transform) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![source_id]) + .name(name) + .transform(transform) + .build() + .unwrap(), + ) .unwrap(); } builder.build().bind(schema.clone()).unwrap() @@ -261,12 +270,15 @@ mod tests { // Spec 0: old name let spec_v0 = PartitionSpec::builder(Arc::new(schema.clone())) .with_spec_id(0) - .add_unbound_field(crate::spec::UnboundPartitionField { - source_id: 4, - field_id: Some(1000), - name: "cat_old".to_string(), - transform: Transform::Identity, - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("cat_old".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -274,12 +286,15 @@ mod tests { // Spec 1: newer name, same field_id let spec_v1 = PartitionSpec::builder(Arc::new(schema.clone())) .with_spec_id(1) - .add_unbound_field(crate::spec::UnboundPartitionField { - source_id: 4, - field_id: Some(1000), - name: "cat_new".to_string(), - transform: Transform::Identity, - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("cat_new".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -297,12 +312,15 @@ mod tests { // Spec 0 (older): category partitioned by identity let spec_v0 = PartitionSpec::builder(Arc::new(schema.clone())) .with_spec_id(0) - .add_unbound_field(crate::spec::UnboundPartitionField { - source_id: 4, - field_id: Some(1000), - name: "category".to_string(), - transform: Transform::Identity, - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("category".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -310,12 +328,15 @@ mod tests { // Spec 1 (newer): same field_id voided (partition dropped) let spec_v1 = PartitionSpec::builder(Arc::new(schema.clone())) .with_spec_id(1) - .add_unbound_field(crate::spec::UnboundPartitionField { - source_id: 4, - field_id: Some(1000), - name: "category_v2".to_string(), - transform: Transform::Void, - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("category_v2".to_string()) + .transform(Transform::Void) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -371,12 +392,15 @@ mod tests { // Spec 1: partition by category + ts_year let spec_v1 = PartitionSpec::builder(Arc::new(schema.clone())) .with_spec_id(1) - .add_unbound_field(crate::spec::UnboundPartitionField { - source_id: 4, - field_id: Some(spec_v0.fields()[0].field_id), - name: "category".to_string(), - transform: Transform::Identity, - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(spec_v0.fields()[0].field_id) + .name("category".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .add_partition_field("ts", "ts_year", Transform::Year) .unwrap() diff --git a/crates/iceberg/src/scan/cache.rs b/crates/iceberg/src/scan/cache.rs index b5eb3c4ce0..b87786e6cf 100644 --- a/crates/iceberg/src/scan/cache.rs +++ b/crates/iceberg/src/scan/cache.rs @@ -270,7 +270,7 @@ mod tests { use crate::expr::{Bind, BoundPredicate, Reference}; use crate::spec::{ Datum, FormatVersion, NestedField, PrimitiveType, Schema, SortOrder, TableMetadataBuilder, - Transform, Type, UnboundPartitionSpec, + Transform, Type, UnboundPartitionField, UnboundPartitionSpec, }; /// A historical spec whose source column was dropped from the current schema resolves to an @@ -288,7 +288,14 @@ mod tests { // Spec 0 partitions on `part` (id 2). let spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(2, "part", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("part") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build(); diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index 3999c0d010..17aa3de968 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -264,20 +264,157 @@ impl PartitionKey { /// Reference to [`UnboundPartitionSpec`]. pub type UnboundPartitionSpecRef = Arc; /// Unbound partition field can be built without a schema and later bound to a schema. +/// +/// The fields are private so that an instance is always well formed: in particular +/// `source_ids` holds at least one id, which [`UnboundPartitionField::builder`] checks when the +/// field is built. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, TypedBuilder)] -#[serde(rename_all = "kebab-case")] +#[serde( + try_from = "self::_serde::UnboundPartitionFieldSerde", + into = "self::_serde::UnboundPartitionFieldSerde" +)] +#[builder(build_method(into = Result))] pub struct UnboundPartitionField { - /// A source column id from the table’s schema - pub source_id: i32, + /// The source column ids from the table’s schema. A single-argument transform reads one + /// id; a v3 multi-argument transform reads several. + source_ids: Vec, /// A partition field id that is used to identify a partition field and is unique within a partition spec. /// In v2 table metadata, it is unique across all partition specs. #[builder(default, setter(strip_option(fallback = field_id_opt)))] - #[serde(skip_serializing_if = "Option::is_none")] - pub field_id: Option, + field_id: Option, /// A partition name. - pub name: String, + #[builder(setter(into))] + name: String, /// A transform that is applied to the source column to produce a partition value. - pub transform: Transform, + transform: Transform, +} + +impl UnboundPartitionField { + /// The single source column id this field reads. + /// + /// Returns an error for a multi-argument field, which reads several columns and therefore + /// has no single source id. Use [`Self::source_ids`] to handle both shapes. + pub fn source_id(&self) -> Result { + match self.source_ids.as_slice() { + [source_id] => Ok(*source_id), + source_ids => Err(invalid_data!( + "Partition field '{}' reads {} source columns and has no single source id", + self.name, + source_ids.len() + )), + } + } + + /// The source column ids this field reads, in order. Never empty. + pub fn source_ids(&self) -> &[i32] { + &self.source_ids + } + + /// The partition field id, when one was assigned. + pub fn field_id(&self) -> Option { + self.field_id + } + + /// The partition name. + pub fn name(&self) -> &str { + &self.name + } + + /// The transform applied to the source columns to produce a partition value. + pub fn transform(&self) -> Transform { + self.transform + } + + /// Return this field with the given partition field id assigned. + pub fn with_field_id(self, field_id: i32) -> Self { + Self { + field_id: Some(field_id), + ..self + } + } + + fn validate(&self) -> Result<()> { + if self.source_ids.is_empty() { + return Err(invalid_data!("Empty source-ids is not allowed")); + } + Ok(()) + } +} + +impl From for Result { + fn from(field: UnboundPartitionField) -> Self { + field.validate()?; + Ok(field) + } +} + +mod _serde { + use serde::{Deserialize, Serialize}; + + use super::UnboundPartitionField; + use crate::Error; + use crate::error::invalid_data; + use crate::spec::Transform; + + /// Per the spec a single-argument field carries `source-id` and a multi-argument field + /// carries `source-ids`. Either spelling is read, but not both at once; the one that + /// matches the field is written. + #[derive(Serialize, Deserialize)] + #[serde(rename_all = "kebab-case")] + pub(super) struct UnboundPartitionFieldSerde { + #[serde(default, skip_serializing_if = "Option::is_none")] + source_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + source_ids: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + field_id: Option, + name: String, + transform: Transform, + } + + impl TryFrom for UnboundPartitionField { + type Error = Error; + + fn try_from(value: UnboundPartitionFieldSerde) -> Result { + let source_ids = match (value.source_id, value.source_ids) { + (Some(source_id), None) => vec![source_id], + (None, Some(source_ids)) => source_ids, + (Some(_), Some(_)) => { + return Err(invalid_data!( + "source-id and source-ids are mutually exclusive" + )); + } + (None, None) => { + return Err(invalid_data!( + "Either `source-id` or `source-ids` must be present" + )); + } + }; + + UnboundPartitionField::builder() + .source_ids(source_ids) + .field_id_opt(value.field_id) + .name(value.name) + .transform(value.transform) + .build() + } + } + + impl From for UnboundPartitionFieldSerde { + fn from(value: UnboundPartitionField) -> Self { + let (source_id, source_ids) = match value.source_ids.as_slice() { + [source_id] => (Some(*source_id), None), + _ => (None, Some(value.source_ids)), + }; + Self { + source_id, + source_ids, + field_id: value.field_id, + name: value.name, + transform: value.transform, + } + } + } } /// Unbound partition spec can be built without a schema and later bound to a schema. @@ -341,7 +478,7 @@ fn has_sequential_ids(field_ids: impl Iterator) -> bool { impl From for UnboundPartitionField { fn from(field: PartitionField) -> Self { UnboundPartitionField { - source_id: field.source_id, + source_ids: vec![field.source_id], field_id: Some(field.field_id), name: field.name, transform: field.transform, @@ -381,19 +518,14 @@ impl UnboundPartitionSpecBuilder { } /// Add a new partition field to the partition spec from an unbound partition field. - pub fn add_partition_field( - self, - source_id: i32, - target_name: impl ToString, - transformation: Transform, - ) -> Result { - let field = UnboundPartitionField { - source_id, - field_id: None, - name: target_name.to_string(), - transform: transformation, - }; - self.add_partition_field_internal(field) + pub fn add_partition_field(mut self, field: UnboundPartitionField) -> Result { + self.check_name_set_and_unique(&field.name)?; + self.check_for_redundant_partitions(&field.source_ids, &field.transform)?; + if let Some(partition_field_id) = field.field_id { + self.check_partition_id_unique(partition_field_id)?; + } + self.fields.push(field); + Ok(self) } /// Add multiple partition fields to the partition spec. @@ -403,21 +535,11 @@ impl UnboundPartitionSpecBuilder { ) -> Result { let mut builder = self; for field in fields { - builder = builder.add_partition_field_internal(field)?; + builder = builder.add_partition_field(field)?; } Ok(builder) } - fn add_partition_field_internal(mut self, field: UnboundPartitionField) -> Result { - self.check_name_set_and_unique(&field.name)?; - self.check_for_redundant_partitions(field.source_id, &field.transform)?; - if let Some(partition_field_id) = field.field_id { - self.check_partition_id_unique(partition_field_id)?; - } - self.fields.push(field); - Ok(self) - } - /// Build the unbound partition spec. pub fn build(self) -> UnboundPartitionSpec { UnboundPartitionSpec { @@ -495,7 +617,7 @@ impl PartitionSpecBuilder { })? .id; let field = UnboundPartitionField { - source_id, + source_ids: vec![source_id], field_id: None, name: target_name.into(), transform, @@ -510,7 +632,7 @@ impl PartitionSpecBuilder { /// Otherwise, a new `field_id` is assigned. pub fn add_unbound_field(mut self, field: UnboundPartitionField) -> Result { self.check_name_set_and_unique(&field.name)?; - self.check_for_redundant_partitions(field.source_id, &field.transform)?; + self.check_for_redundant_partitions(&field.source_ids, &field.transform)?; Self::check_name_does_not_collide_with_schema(&field, &self.schema)?; Self::check_transform_compatibility(&field, &self.schema)?; if let Some(partition_field_id) = field.field_id { @@ -574,7 +696,7 @@ impl PartitionSpecBuilder { }; bound_fields.push(PartitionField { - source_id: field.source_id, + source_id: field.source_id()?, field_id: partition_field_id, name: field.name, transform: field.transform, @@ -595,14 +717,14 @@ impl PartitionSpecBuilder { match schema.field_by_name(field.name.as_str()) { Some(schema_collision) => { if field.transform == Transform::Identity { - if schema_collision.id == field.source_id { + if schema_collision.id == field.source_id()? { Ok(()) } else { Err(invalid_data!( "Cannot create identity partition sourced from different field in schema. Field name '{}' has id `{}` in schema but partition source id is `{}`", field.name, schema_collision.id, - field.source_id + field.source_id()? )) } } else { @@ -619,11 +741,11 @@ impl PartitionSpecBuilder { /// Ensure that the transformation of the field is compatible with type of the field /// in the schema. Implicitly also checks if the source field exists in the schema. fn check_transform_compatibility(field: &UnboundPartitionField, schema: &Schema) -> Result<()> { - let schema_field = schema.field_by_id(field.source_id).ok_or_else(|| { - invalid_data!( - "Cannot find partition source field with id `{}` in schema", - field.source_id - ) + // A multi-argument field is rejected here on purpose: a bound `PartitionField` has + // no place for more than one source id yet. + let source_id = field.source_id()?; + let schema_field = schema.field_by_id(source_id).ok_or_else(|| { + invalid_data!("Cannot find partition source field with id `{source_id}` in schema") })?; if field.transform != Transform::Void { @@ -668,15 +790,19 @@ trait CorePartitionSpecValidator { } /// For a single source-column transformations must be unique. - fn check_for_redundant_partitions(&self, source_id: i32, transform: &Transform) -> Result<()> { + fn check_for_redundant_partitions( + &self, + source_ids: &[i32], + transform: &Transform, + ) -> Result<()> { let collision = self.fields().iter().find(|f| { - f.source_id == source_id && f.transform.dedup_name() == transform.dedup_name() + f.source_ids == source_ids && f.transform.dedup_name() == transform.dedup_name() }); if let Some(collision) = collision { Err(invalid_data!( - "Cannot add redundant partition with source id `{}` and transform `{}`. A partition with the same source id and transform already exists with name `{}`", - source_id, + "Cannot add redundant partition with source ids `{:?}` and transform `{}`. A partition with the same source ids and transform already exists with name `{}`", + source_ids, transform.dedup_name(), collision.name )) @@ -779,15 +905,17 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("id".to_string()) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), UnboundPartitionField::builder() - .source_id(2) + .source_ids(vec![2]) .name("name_string".to_string()) .transform(Transform::Void) - .build(), + .build() + .unwrap(), ]) .unwrap() .with_spec_id(1) @@ -802,15 +930,17 @@ mod tests { .with_spec_id(1) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("id_void".to_string()) .transform(Transform::Void) - .build(), + .build() + .unwrap(), UnboundPartitionField::builder() - .source_id(2) + .source_ids(vec![2]) .name("name_void".to_string()) .transform(Transform::Void) - .build(), + .build() + .unwrap(), ]) .unwrap() .build() @@ -848,17 +978,17 @@ mod tests { let partition_spec: UnboundPartitionSpec = serde_json::from_str(spec).unwrap(); assert_eq!(Some(1), partition_spec.spec_id); - assert_eq!(4, partition_spec.fields[0].source_id); + assert_eq!(4, partition_spec.fields[0].source_id().unwrap()); assert_eq!(Some(1000), partition_spec.fields[0].field_id); assert_eq!("ts_day", partition_spec.fields[0].name); assert_eq!(Transform::Day, partition_spec.fields[0].transform); - assert_eq!(1, partition_spec.fields[1].source_id); + assert_eq!(1, partition_spec.fields[1].source_id().unwrap()); assert_eq!(Some(1001), partition_spec.fields[1].field_id); assert_eq!("id_bucket", partition_spec.fields[1].name); assert_eq!(Transform::Bucket(16), partition_spec.fields[1].transform); - assert_eq!(2, partition_spec.fields[2].source_id); + assert_eq!(2, partition_spec.fields[2].source_id().unwrap()); assert_eq!(Some(1002), partition_spec.fields[2].field_id); assert_eq!("id_truncate", partition_spec.fields[2].name); assert_eq!(Transform::Truncate(4), partition_spec.fields[2].transform); @@ -875,7 +1005,7 @@ mod tests { let partition_spec: UnboundPartitionSpec = serde_json::from_str(spec).unwrap(); assert_eq!(None, partition_spec.spec_id); - assert_eq!(4, partition_spec.fields[0].source_id); + assert_eq!(4, partition_spec.fields[0].source_id().unwrap()); assert_eq!(None, partition_spec.fields[0].field_id); assert_eq!("ts_day", partition_spec.fields[0].name); assert_eq!(Transform::Day, partition_spec.fields[0].transform); @@ -884,7 +1014,14 @@ mod tests { #[test] fn test_unbound_partition_spec_serialization_skips_none_fields() { let spec = UnboundPartitionSpec::builder() - .add_partition_field(4, "ts_day".to_string(), Transform::Day) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -1131,9 +1268,23 @@ mod tests { #[test] fn test_builder_disallow_duplicate_names() { UnboundPartitionSpec::builder() - .add_partition_field(1, "ts_day".to_string(), Transform::Day) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap() - .add_partition_field(2, "ts_day".to_string(), Transform::Day) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap_err(); } @@ -1148,14 +1299,14 @@ mod tests { .unwrap(); PartitionSpec::builder(schema.clone()) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(1000), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1000), name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1176,14 +1327,14 @@ mod tests { let spec = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], name: "id".to_string(), transform: Transform::Identity, field_id: Some(1012), }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], name: "name_void".to_string(), transform: Transform::Void, field_id: None, @@ -1191,7 +1342,7 @@ mod tests { .unwrap() // Should keep its ID even if its lower .add_unbound_field(UnboundPartitionField { - source_id: 3, + source_ids: vec![3], name: "year".to_string(), transform: Transform::Year, field_id: Some(1), @@ -1262,7 +1413,7 @@ mod tests { let err = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id".to_string(), transform: Transform::Bucket(16), @@ -1289,7 +1440,7 @@ mod tests { PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id".to_string(), transform: Transform::Identity, @@ -1302,7 +1453,7 @@ mod tests { PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "id".to_string(), transform: Transform::Identity, @@ -1326,13 +1477,13 @@ mod tests { .with_spec_id(1) .add_unbound_fields(vec![ UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), }, UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "name".to_string(), transform: Transform::Identity, @@ -1347,13 +1498,13 @@ mod tests { .with_spec_id(1) .add_unbound_fields(vec![ UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), }, UnboundPartitionField { - source_id: 4, + source_ids: vec![4], field_id: None, name: "name".to_string(), transform: Transform::Identity, @@ -1375,7 +1526,7 @@ mod tests { let err = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_fields(vec![UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "v_part".to_string(), transform: Transform::Identity, @@ -1392,12 +1543,22 @@ mod tests { fn test_builder_disallows_redundant() { let err = UnboundPartitionSpec::builder() .with_spec_id(1) - .add_partition_field(1, "id_bucket[16]".to_string(), Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket[16]") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .add_partition_field( - 1, - "id_bucket_with_other_name".to_string(), - Transform::Bucket(16), + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket_with_other_name") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), ) .unwrap_err(); assert!(err.message().contains("redundant partition")); @@ -1415,7 +1576,7 @@ mod tests { PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_year".to_string(), transform: Transform::Year, @@ -1428,7 +1589,7 @@ mod tests { let spec = UnboundPartitionSpec::builder() .with_spec_id(1) .add_partition_fields(vec![UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket[16]".to_string(), transform: Transform::Bucket(16), @@ -1439,7 +1600,7 @@ mod tests { assert_eq!(spec, UnboundPartitionSpec { spec_id: Some(1), fields: vec![UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket[16]".to_string(), transform: Transform::Bucket(16), @@ -1460,7 +1621,7 @@ mod tests { let partition_spec_1 = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1472,7 +1633,7 @@ mod tests { let partition_spec_2 = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1496,7 +1657,7 @@ mod tests { let partition_spec_1 = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1508,7 +1669,7 @@ mod tests { let partition_spec_2 = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(32), @@ -1533,7 +1694,7 @@ mod tests { let partition_spec_1 = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1545,7 +1706,7 @@ mod tests { let partition_spec_2 = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1570,14 +1731,14 @@ mod tests { let partition_spec_1 = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "name".to_string(), transform: Transform::Identity, @@ -1589,14 +1750,14 @@ mod tests { let partition_spec_2 = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "name".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1631,14 +1792,14 @@ mod tests { let spec = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(1001), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1000), name: "name".to_string(), transform: Transform::Identity, @@ -1663,14 +1824,14 @@ mod tests { let spec = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(1000), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1001), name: "name".to_string(), transform: Transform::Identity, @@ -1697,14 +1858,14 @@ mod tests { let spec = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(999), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1000), name: "name".to_string(), transform: Transform::Identity, @@ -1731,14 +1892,14 @@ mod tests { let spec = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(1000), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1002), name: "name".to_string(), transform: Transform::Identity, @@ -1847,4 +2008,106 @@ mod tests { "data=a%2Fb%2Fc%2Fd/data_truc_10=a%2Fb%2Fc%2Fd" ); } + + #[test] + fn test_unbound_partition_field_reads_source_ids_only() { + let field: UnboundPartitionField = serde_json::from_str( + r#"{"source-ids": [1, 2], "name": "m", "transform": "bucket[4]"}"#, + ) + .unwrap(); + + assert_eq!([1, 2], field.source_ids()); + // a multi-argument field has no single source id + assert!( + field + .source_id() + .unwrap_err() + .to_string() + .contains("has no single source id") + ); + + let serialized = serde_json::to_value(&field).unwrap(); + assert_eq!( + Some(&serde_json::json!([1, 2])), + serialized.get("source-ids") + ); + assert!(serialized.get("source-id").is_none()); + } + + #[test] + fn test_unbound_partition_field_single_source_id_round_trip() { + let field: UnboundPartitionField = + serde_json::from_str(r#"{"source-id": 1, "name": "m", "transform": "identity"}"#) + .unwrap(); + + assert_eq!(1, field.source_id().unwrap()); + assert_eq!([1], field.source_ids()); + + // a single-argument field keeps writing source-id + let serialized = serde_json::to_value(&field).unwrap(); + assert_eq!(Some(&serde_json::json!(1)), serialized.get("source-id")); + assert!(serialized.get("source-ids").is_none()); + } + + #[test] + fn test_unbound_partition_field_rejects_malformed_source_ids() { + for (input, expected) in [ + ( + r#"{"source-ids": [], "name": "m", "transform": "identity"}"#, + "Empty source-ids is not allowed", + ), + ( + r#"{"name": "m", "transform": "identity"}"#, + "Either `source-id` or `source-ids` must be present", + ), + ( + r#"{"source-id": 1, "source-ids": [1], "name": "m", "transform": "identity"}"#, + "mutually exclusive", + ), + ] { + let err = serde_json::from_str::(input).unwrap_err(); + assert!( + err.to_string().contains(expected), + "unexpected error for {input}: {err}" + ); + } + } + + #[test] + fn test_binding_a_multi_argument_field_fails_loudly() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "b", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let spec: UnboundPartitionSpec = serde_json::from_str( + r#"{"spec-id": 1, "fields": [{"source-ids": [1, 2], "field-id": 1000, "name": "m", "transform": "bucket[4]"}]}"#, + ) + .unwrap(); + + // A bound PartitionField still carries a single source id, so binding has to refuse + // rather than quietly keep the first one. + let err = spec.bind(schema).unwrap_err(); + assert!( + err.to_string().contains("has no single source id"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_unbound_partition_field_builder_rejects_empty_source_ids() { + let err = UnboundPartitionField::builder() + .source_ids(vec![]) + .name("m") + .transform(Transform::Identity) + .build() + .unwrap_err(); + assert!( + err.to_string().contains("Empty source-ids is not allowed"), + "unexpected error: {err}" + ); + } } diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index a48a5abdda..f8e69c4c86 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -828,10 +828,11 @@ mod tests { PartitionSpec::builder(schema.clone()) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(2) + .source_ids(vec![2]) .name("year".to_string()) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ]) .unwrap() .with_spec_id(1) @@ -978,10 +979,11 @@ mod tests { PartitionSpec::builder(schema.clone()) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(2) + .source_ids(vec![2]) .name("year".to_string()) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ]) .unwrap() .with_spec_id(1) diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 24f10355bf..83468602cd 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -1721,12 +1721,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "ts_day".to_string(), - transform: Transform::Day, - source_id: 4, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("ts_day".to_string()) + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -1869,12 +1872,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "ts_day".to_string(), - transform: Transform::Day, - source_id: 4, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("ts_day".to_string()) + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -2945,12 +2951,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "x".to_string(), - transform: Transform::Identity, - source_id: 1, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1000) + .name("x".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -3042,12 +3051,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema2.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "x".to_string(), - transform: Transform::Identity, - source_id: 1, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1000) + .name("x".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -3171,12 +3183,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "x".to_string(), - transform: Transform::Identity, - source_id: 1, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1000) + .name("x".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -3257,12 +3272,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "x".to_string(), - transform: Transform::Identity, - source_id: 1, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1000) + .name("x".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -4481,7 +4499,14 @@ mod tests { schema.clone(), UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(2, "y", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("y") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build(), SortOrder::unsorted_order(), @@ -4496,7 +4521,14 @@ mod tests { .into_builder(None) .add_partition_spec( UnboundPartitionSpec::builder() - .add_partition_field(3, "z", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![3]) + .name("z") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build(), ) diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 3ed848d2d6..9a8206cb27 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -741,7 +741,7 @@ impl TableMetadataBuilder { for partition_field in unbound_spec.fields() { let exists_in_any_schema = self .metadata - .name_exists_in_any_schema(&partition_field.name); + .name_exists_in_any_schema(partition_field.name()); // Skip if partition field name doesn't conflict with any schema field if !exists_in_any_schema { @@ -749,15 +749,15 @@ impl TableMetadataBuilder { } // If name exists in schemas, validate against current schema rules - if let Some(schema_field) = current_schema.field_by_name(&partition_field.name) { + if let Some(schema_field) = current_schema.field_by_name(partition_field.name()) { let is_identity_transform = - partition_field.transform == crate::spec::Transform::Identity; - let has_matching_source_id = schema_field.id == partition_field.source_id; + partition_field.transform() == crate::spec::Transform::Identity; + let has_matching_source_id = schema_field.id == partition_field.source_id()?; if !is_identity_transform { return Err(invalid_data!( "Cannot create partition with name '{}' that conflicts with schema field and is not an identity transform.", - partition_field.name + partition_field.name() )); } @@ -765,9 +765,9 @@ impl TableMetadataBuilder { return Err(invalid_data!( "Cannot create identity partition sourced from different field in schema. \ Field name '{}' has id `{}` in schema but partition source id is `{}`", - partition_field.name, + partition_field.name(), schema_field.id, - partition_field.source_id + partition_field.source_id()? )); } } @@ -850,21 +850,22 @@ impl TableMetadataBuilder { .partition_specs .values() .flat_map(|spec| spec.fields()) - .map(|field| ((field.source_id, &field.transform), field.field_id)) + .map(|field| ((vec![field.source_id], field.transform), field.field_id)) .collect(); // Create new fields with reused field IDs where possible let fields = unbound_spec .fields .into_iter() - .map(|mut field| { - if field.field_id.is_none() + .map(|field| { + if field.field_id().is_none() && let Some(&existing_field_id) = - equivalent_field_ids.get(&(field.source_id, &field.transform)) + equivalent_field_ids.get(&(field.source_ids().to_vec(), field.transform())) { - field.field_id = Some(existing_field_id); + field.with_field_id(existing_field_id) + } else { + field } - field }) .collect(); @@ -1228,15 +1229,19 @@ impl TableMetadataBuilder { // Re-build partition spec with new ids let mut fresh_spec = PartitionSpecBuilder::new(fresh_schema.clone()); for field in spec.fields() { - let source_field_name = previous_id_to_name.get(&field.source_id).ok_or_else(|| { + let source_id = field.source_id()?; + let source_field_name = previous_id_to_name.get(&source_id).ok_or_else(|| { invalid_data!( "Cannot find source column with id {} for partition column {} in schema.", - field.source_id, - field.name + source_id, + field.name() ) })?; - fresh_spec = - fresh_spec.add_partition_field(source_field_name, &field.name, field.transform)?; + fresh_spec = fresh_spec.add_partition_field( + source_field_name, + field.name(), + field.transform(), + )?; } let fresh_spec = fresh_spec.build()?; @@ -1432,7 +1437,14 @@ mod tests { fn partition_spec() -> UnboundPartitionSpec { UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(2, "y", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("y") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() } @@ -1666,12 +1678,15 @@ mod tests { // partition_spec() has None set for field-id spec: PartitionSpec::builder(schema()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "y".to_string(), - transform: Transform::Identity, - source_id: 2, - field_id: Some(1000) - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("y".to_string()) + .transform(Transform::Identity) + .build() + .unwrap() + ) .unwrap() .build() .unwrap() @@ -1739,20 +1754,21 @@ mod tests { let added_spec = UnboundPartitionSpec::builder() .with_spec_id(10) .add_partition_fields(vec![ - UnboundPartitionField { - // The previous field - has field_id set - name: "y".to_string(), - transform: Transform::Identity, - source_id: 2, - field_id: Some(1000), - }, - UnboundPartitionField { - // A new field without field id - should still be without field id in changes - name: "z".to_string(), - transform: Transform::Identity, - source_id: 3, - field_id: None, - }, + // The previous field - has field_id set + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("y".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + // A new field without field id - should still be without field id in changes + UnboundPartitionField::builder() + .source_ids(vec![3]) + .name("z".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), ]) .unwrap() .build(); @@ -1767,19 +1783,25 @@ mod tests { let expected_change = added_spec.with_spec_id(1); let expected_spec = PartitionSpec::builder(schema()) .with_spec_id(1) - .add_unbound_field(UnboundPartitionField { - name: "y".to_string(), - transform: Transform::Identity, - source_id: 2, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("y".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() - .add_unbound_field(UnboundPartitionField { - name: "z".to_string(), - transform: Transform::Identity, - source_id: 3, - field_id: Some(1001), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![3]) + .field_id(1001) + .name("z".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -1817,7 +1839,14 @@ mod tests { let schema = builder.get_current_schema().unwrap().clone(); let added_spec = UnboundPartitionSpec::builder() .with_spec_id(10) - .add_partition_field(1, "y_bucket[2]", Transform::Bucket(2)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("y_bucket[2]") + .transform(Transform::Bucket(2)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -1831,12 +1860,15 @@ mod tests { let expected_spec = PartitionSpec::builder(schema) .with_spec_id(1) - .add_unbound_field(UnboundPartitionField { - name: "y_bucket[2]".to_string(), - transform: Transform::Bucket(2), - source_id: 1, - field_id: Some(1001), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1001) + .name("y_bucket[2]".to_string()) + .transform(Transform::Bucket(2)) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -2401,18 +2433,20 @@ mod tests { let added_spec = UnboundPartitionSpec::builder() .with_spec_id(10) .add_partition_fields(vec![ - UnboundPartitionField { - name: "y".to_string(), - transform: Transform::Identity, - source_id: 2, - field_id: Some(1000), - }, - UnboundPartitionField { - name: "z".to_string(), - transform: Transform::Identity, - source_id: 3, - field_id: Some(1002), - }, + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("y".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + UnboundPartitionField::builder() + .source_ids(vec![3]) + .field_id(1002) + .name("z".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), ]) .unwrap() .build(); @@ -2737,7 +2771,14 @@ mod tests { let partition_spec_with_bucket = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "bucket_data", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("bucket_data") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2796,7 +2837,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "partition_col", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("partition_col") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2845,7 +2893,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "data_bucket", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("data_bucket") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2868,7 +2923,14 @@ mod tests { let conflicting_partition_spec = UnboundPartitionSpec::builder() .with_spec_id(1) - .add_partition_field(1, "existing_field", Transform::Bucket(8)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("existing_field") + .transform(Transform::Bucket(8)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2898,7 +2960,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "bucket_data", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("bucket_data") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2994,7 +3063,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(2, "partition_data", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("partition_data") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -3043,7 +3119,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "bucket_data", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("bucket_data") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -3093,7 +3176,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "data_bucket", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("data_bucket") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -3117,7 +3207,14 @@ mod tests { // Try to add a partition spec with a field name that does NOT conflict with existing schema fields let non_conflicting_partition_spec = UnboundPartitionSpec::builder() .with_spec_id(1) - .add_partition_field(2, "new_partition_field", Transform::Bucket(8)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("new_partition_field") + .transform(Transform::Bucket(8)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -3518,7 +3615,14 @@ mod tests { // Create initial table with spec 0: identity(id) -> field_id = 1000 let initial_spec = UnboundPartitionSpec::builder() - .add_partition_field(1, "id", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -3537,7 +3641,14 @@ mod tests { // Add spec 1: bucket(data) -> field_id = 1001 let spec1 = UnboundPartitionSpec::builder() - .add_partition_field(2, "data_bucket", Transform::Bucket(10)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("data_bucket") + .transform(Transform::Bucket(10)) + .build() + .unwrap(), + ) .unwrap() .build(); let builder = metadata.into_builder(Some("s3://bucket/table/metadata/v1.json".to_string())); @@ -3547,11 +3658,32 @@ mod tests { // Add spec 2: identity(id) + bucket(data) + year(timestamp) // Should reuse field_id 1000 for identity(id) and 1001 for bucket(data) let spec2 = UnboundPartitionSpec::builder() - .add_partition_field(1, "id", Transform::Identity) // Should reuse 1000 - .unwrap() - .add_partition_field(2, "data_bucket", Transform::Bucket(10)) // Should reuse 1001 - .unwrap() - .add_partition_field(3, "year", Transform::Year) // Should get new 1002 + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id") + .transform(Transform::Identity) + .build() + .unwrap(), + ) // Should reuse 1000 + .unwrap() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("data_bucket") + .transform(Transform::Bucket(10)) + .build() + .unwrap(), + ) // Should reuse 1001 + .unwrap() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![3]) + .name("year") + .transform(Transform::Year) + .build() + .unwrap(), + ) // Should get new 1002 .unwrap() .build(); let builder = metadata.into_builder(Some("s3://bucket/table/metadata/v2.json".to_string())); From 71e571e531618371499eaf3c995e30c69910e2d9 Mon Sep 17 00:00:00 2001 From: kamcheungting-db <91572897+kamcheungting-db@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:24:54 +0000 Subject: [PATCH 21/63] refactor(reader): build Arrow reader metadata once (#3242) * refactor(reader): build Arrow reader metadata once * refactor(reader): address review feedback on metadata rebuild - Defer materializing an owned Arrow schema until a rebuild is actually needed; the embedded-IDs/no-INT96/no-row-number fast path now returns early without cloning. - Interpolate missing_field_ids, install_row_number, and the resulting schema into the ArrowReaderMetadata error message so failures identify which transform was active. - Add a regression test covering the combined path: a field-id-less file with INT96 timestamps and a _pos projection, asserting coerced timestamps and the row-number column together. Co-authored-by: Isaac * refactor(reader): address remaining review feedback --------- Co-authored-by: Isaac --- crates/iceberg/src/arrow/reader/pipeline.rs | 256 +++++++++++++------- 1 file changed, 162 insertions(+), 94 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index ee027c5e72..11fa200ee9 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -171,74 +171,6 @@ impl FileScanTaskReader { // (see #2403). let use_position_fallback = missing_field_ids && task.name_mapping().is_none(); - // Three-branch schema resolution strategy matching Java's ReadConf constructor - // - // Per Iceberg spec Column Projection rules: - // "Columns in Iceberg data files are selected by field id. The table schema's column - // names and order may change after a data file is written, and projection must be done - // using field ids." - // https://iceberg.apache.org/spec/#column-projection - // - // When Parquet files lack field IDs (e.g., Hive/Spark migrations via add_files), - // we must assign field IDs BEFORE reading data to enable correct projection. - // - // Java's ReadConf determines field ID strategy: - // - Branch 1: hasIds(fileSchema) → trust embedded field IDs, use pruneColumns() - // - Branch 2: nameMapping present → applyNameMapping(), then pruneColumns() - // - Branch 3: fallback → addFallbackIds(), then pruneColumnsFallback() - let arrow_metadata = if missing_field_ids { - // Parquet file lacks field IDs - must assign them before reading - let arrow_schema = if let Some(name_mapping) = task.name_mapping() { - // Branch 2: Apply name mapping to assign correct Iceberg field IDs - // Per spec rule #2: "Use schema.name-mapping.default metadata to map field id - // to columns without field id" - // Corresponds to Java's ParquetSchemaUtil.applyNameMapping() - apply_name_mapping_to_arrow_schema( - Arc::clone(arrow_metadata.schema()), - name_mapping, - )? - } else { - // Branch 3: No name mapping - use position-based fallback IDs - // Corresponds to Java's ParquetSchemaUtil.addFallbackIds() - add_fallback_field_ids_to_arrow_schema(arrow_metadata.schema()) - }; - - let options = ArrowReaderOptions::new().with_schema(arrow_schema); - ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err( - |e| { - Error::new( - ErrorKind::Unexpected, - "Failed to create ArrowReaderMetadata with field ID schema", - ) - .with_source(e) - }, - )? - } else { - // Branch 1: File has embedded field IDs - trust them - arrow_metadata - }; - - // Coerce INT96 timestamp columns to the resolution specified by the Iceberg schema. - // This must happen before building the stream reader to avoid i64 overflow in arrow-rs. - let arrow_metadata = if let Some(coerced_schema) = - coerce_int96_timestamps(arrow_metadata.schema(), task.schema()) - { - let options = ArrowReaderOptions::new().with_schema(Arc::clone(&coerced_schema)); - ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err( - |e| { - Error::new( - ErrorKind::Unexpected, - format!( - "Failed to create ArrowReaderMetadata with INT96-coerced schema: {coerced_schema}" - ), - ) - .with_source(e) - }, - )? - } else { - arrow_metadata - }; - let project_pos = task.project_field_ids().contains(&RESERVED_FIELD_ID_POS); let project_row_id = task.project_field_ids().contains(&RESERVED_FIELD_ID_ROW_ID); @@ -256,32 +188,12 @@ impl FileScanTaskReader { let install_row_number = need_row_number || metadata_only_projection; - let arrow_metadata = if install_row_number { - let row_number_field = Arc::new( - Field::new(RESERVED_COL_NAME_POS, DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - RESERVED_FIELD_ID_POS.to_string(), - )])) - .with_extension_type(RowNumber), - ); - - let options = ArrowReaderOptions::new() - .with_schema(Arc::clone(arrow_metadata.schema())) - .with_virtual_columns(vec![row_number_field])?; - - ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err( - |e| { - Error::new( - ErrorKind::Unexpected, - "Failed to create ArrowReaderMetadata with the 'row_number' virtual_column", - ) - .with_source(e) - }, - )? - } else { - arrow_metadata - }; + let arrow_metadata = Self::configure_arrow_reader_metadata( + arrow_metadata, + &task, + missing_field_ids, + install_row_number, + )?; // Build the stream reader, reusing the already-opened file reader let mut record_batch_stream_builder = @@ -720,6 +632,71 @@ impl FileScanTaskReader { Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream) } + /// Applies all task-specific schema and virtual-column options, rebuilding the + /// Arrow reader metadata at most once. + fn configure_arrow_reader_metadata( + arrow_metadata: ArrowReaderMetadata, + task: &FileScanTask, + missing_field_ids: bool, + install_row_number: bool, + ) -> Result { + // Schema resolution follows the Iceberg Column Projection rule: + // "Columns in Iceberg data files are selected by field id." + // https://iceberg.apache.org/spec/#column-projection + // + // This mirrors Java's ReadConf strategy: use embedded IDs with pruneColumns(); + // otherwise, use applyNameMapping() followed by pruneColumns() when configured, or + // addFallbackIds() followed by pruneColumnsFallback() for position-based fallback. + // The fast path returns early without materializing an owned schema. + let arrow_schema = if missing_field_ids { + let schema = if let Some(name_mapping) = task.name_mapping() { + apply_name_mapping_to_arrow_schema( + Arc::clone(arrow_metadata.schema()), + name_mapping, + )? + } else { + add_fallback_field_ids_to_arrow_schema(arrow_metadata.schema()) + }; + // Coerce INT96 timestamp columns before building the stream reader to avoid + // i64 overflow in arrow-rs. Apply this after assigning any missing field IDs + // so the final schema contains both changes. + coerce_int96_timestamps(&schema, task.schema()).unwrap_or(schema) + } else if let Some(coerced) = + coerce_int96_timestamps(arrow_metadata.schema(), task.schema()) + { + coerced + } else if install_row_number { + Arc::clone(arrow_metadata.schema()) + } else { + return Ok(arrow_metadata); + }; + + let mut options = ArrowReaderOptions::new().with_schema(Arc::clone(&arrow_schema)); + if install_row_number { + let row_number_field = Arc::new( + Field::new(RESERVED_COL_NAME_POS, DataType::Int64, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_POS.to_string(), + )])) + .with_extension_type(RowNumber), + ); + options = options.with_virtual_columns(vec![row_number_field])?; + } + + ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!( + "Failed to create ArrowReaderMetadata with the configured reader options \ + (missing_field_ids: {missing_field_ids}, \ + install_row_number: {install_row_number}, schema: {arrow_schema})" + ), + ) + .with_source(e) + }) + } + /// Reads bloom filters for relevant columns and evaluates the predicate /// against them to filter out row groups that definitely don't match. async fn filter_row_groups_by_bloom_filter( @@ -2844,6 +2821,97 @@ mod tests { assert_int96_read_matches(&file_path, schema, vec![1, 2], &expected_micros).await; } + #[tokio::test] + async fn test_read_int96_timestamps_with_fallback_ids_and_pos() { + use arrow_array::TimestampMicrosecondArray; + + // Regression test for the combined path this refactor introduced: a field-id-less + // file (positional fallback IDs) with an INT96 column and a `_pos` projection. + // All three transforms -- field-ID assignment, INT96 coercion, and the row-number + // virtual column -- apply in the single ArrowReaderMetadata rebuild. + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let (file_path, expected_micros) = + write_int96_parquet_file(&table_location, "no_ids_with_pos.parquet", false); + + let batches = + read_int96_batches(&file_path, schema, vec![1, 2, RESERVED_FIELD_ID_POS]).await; + + assert_eq!(batches.len(), 1); + // The INT96 timestamps are coerced to micros... + let ts_col = batches[0] + .column_by_name("ts") + .expect("ts column should be present") + .as_any() + .downcast_ref::() + .expect("Expected TimestampMicrosecondArray"); + for (i, expected) in expected_micros.iter().enumerate() { + assert_eq!(ts_col.value(i), *expected, "Row {i}"); + } + // ...and `_pos`, materialized by the RowNumber virtual column, counts rows 0,1,2. + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + } + + #[tokio::test] + async fn test_read_int96_timestamps_with_field_ids_and_pos() { + use arrow_array::TimestampMicrosecondArray; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let (file_path, expected_micros) = + write_int96_parquet_file(&table_location, "with_ids_with_pos.parquet", true); + + let batches = + read_int96_batches(&file_path, schema, vec![1, 2, RESERVED_FIELD_ID_POS]).await; + + assert_eq!(batches.len(), 1); + let ts_col = batches[0] + .column_by_name("ts") + .expect("ts column should be present") + .as_any() + .downcast_ref::() + .expect("Expected TimestampMicrosecondArray"); + + for (i, expected) in expected_micros.iter().enumerate() { + assert_eq!(ts_col.value(i), *expected, "Row {i}"); + } + + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + + assert_eq!(pos_col.values(), &[0, 1, 2]); + } + #[tokio::test] async fn test_read_int96_timestamps_in_struct() { use arrow_array::{StructArray, TimestampMicrosecondArray}; From 36eae2969b0773ee5b6f23d9be230fd0a17ffe92 Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 30 Sep 2026 19:25:21 +0000 Subject: [PATCH 22/63] refactor(scan) move test utils to test_utils.rs (#3107) * refactor(scan) move test utils to test_utils.rs * move things around * move * fix * comments --- Cargo.lock | 1 + crates/catalog/rest/src/catalog.rs | 2 +- crates/iceberg/Cargo.toml | 1 - crates/iceberg/public-api.txt | 4 - crates/iceberg/src/inspect/history.rs | 2 +- crates/iceberg/src/inspect/manifests.rs | 2 +- crates/iceberg/src/inspect/snapshots.rs | 2 +- crates/iceberg/src/lib.rs | 3 +- crates/iceberg/src/scan/mod.rs | 1157 +--------------- crates/iceberg/src/test_utils.rs | 190 --- .../iceberg/src/test_utils/delete_vector.rs | 32 + crates/iceberg/src/test_utils/encryption.rs | 94 ++ crates/iceberg/src/test_utils/mod.rs | 33 + crates/iceberg/src/test_utils/record_batch.rs | 76 ++ crates/iceberg/src/test_utils/runtime.rs | 45 + crates/iceberg/src/test_utils/scan.rs | 1160 +++++++++++++++++ crates/iceberg/src/util/snapshot.rs | 2 +- crates/test_utils/Cargo.toml | 1 + crates/test_utils/src/lib.rs | 5 + crates/test_utils/src/runtime.rs | 40 + 20 files changed, 1503 insertions(+), 1349 deletions(-) delete mode 100644 crates/iceberg/src/test_utils.rs create mode 100644 crates/iceberg/src/test_utils/delete_vector.rs create mode 100644 crates/iceberg/src/test_utils/encryption.rs create mode 100644 crates/iceberg/src/test_utils/mod.rs create mode 100644 crates/iceberg/src/test_utils/record_batch.rs create mode 100644 crates/iceberg/src/test_utils/runtime.rs create mode 100644 crates/iceberg/src/test_utils/scan.rs create mode 100644 crates/test_utils/src/runtime.rs diff --git a/Cargo.lock b/Cargo.lock index 64c5c61875..fcf5dd4497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2952,6 +2952,7 @@ name = "iceberg_test_utils" version = "0.10.1" dependencies = [ "iceberg", + "tokio", "tracing-subscriber", ] diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index b7d3b3163c..cc5973316a 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -1673,8 +1673,8 @@ mod tests { SnapshotLog, SortDirection, SortField, SortOrder, Summary, Transform, Type, UnboundPartitionField, UnboundPartitionSpec, }; - use iceberg::test_utils::test_runtime; use iceberg::transaction::{ApplyTransactionAction, Transaction}; + use iceberg_test_utils::test_runtime; use mockito::{Mock, Server, ServerGuard}; use serde_json::json; use uuid::uuid; diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 18256d362b..8783251fd5 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -54,7 +54,6 @@ bytes = { workspace = true } chrono = { workspace = true } crc32fast = { workspace = true } derive_builder = { workspace = true } -expect-test = { workspace = true } fastnum = { workspace = true } flate2 = { workspace = true } fnv = { workspace = true } diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 58c7db7dd6..da0b6b1912 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3229,10 +3229,6 @@ pub fn iceberg::table::TableBuilder::metadata>(self, metadata_location: T) -> Self pub fn iceberg::table::TableBuilder::readonly(self, readonly: bool) -> Self pub fn iceberg::table::TableBuilder::runtime(self, runtime: iceberg::Runtime) -> Self -pub mod iceberg::test_utils -pub fn iceberg::test_utils::check_record_batches(record_batches: alloc::vec::Vec, expected_schema: expect_test::Expect, expected_data: expect_test::Expect, ignore_check_columns: &[&str], sort_column: core::option::Option<&str>) -pub async fn iceberg::test_utils::make_encrypted_table() -> iceberg::table::Table -pub fn iceberg::test_utils::test_runtime() -> iceberg::Runtime pub mod iceberg::transaction pub struct iceberg::transaction::ActionCommit impl iceberg::transaction::ActionCommit diff --git a/crates/iceberg/src/inspect/history.rs b/crates/iceberg/src/inspect/history.rs index fef9b5d6a3..8fec7d5776 100644 --- a/crates/iceberg/src/inspect/history.rs +++ b/crates/iceberg/src/inspect/history.rs @@ -126,9 +126,9 @@ mod tests { use crate::TableIdent; use crate::io::FileIO; - use crate::scan::tests::TableTestFixture; use crate::spec::TableMetadata; use crate::table::Table; + use crate::test_utils::scan::TableTestFixture; use crate::test_utils::{check_record_batches, test_runtime}; fn table_from_metadata_json(metadata_json: &str) -> Table { diff --git a/crates/iceberg/src/inspect/manifests.rs b/crates/iceberg/src/inspect/manifests.rs index e260375769..c9f4c3f403 100644 --- a/crates/iceberg/src/inspect/manifests.rs +++ b/crates/iceberg/src/inspect/manifests.rs @@ -289,9 +289,9 @@ mod tests { use expect_test::expect; use futures::TryStreamExt; - use crate::scan::tests::TableTestFixture; use crate::spec::TableMetadata; use crate::test_utils::check_record_batches; + use crate::test_utils::scan::TableTestFixture; #[tokio::test] async fn test_manifests_table() { diff --git a/crates/iceberg/src/inspect/snapshots.rs b/crates/iceberg/src/inspect/snapshots.rs index fbed7ec11e..22046daba7 100644 --- a/crates/iceberg/src/inspect/snapshots.rs +++ b/crates/iceberg/src/inspect/snapshots.rs @@ -139,8 +139,8 @@ mod tests { use expect_test::expect; use futures::TryStreamExt; - use crate::scan::tests::TableTestFixture; use crate::test_utils::check_record_batches; + use crate::test_utils::scan::TableTestFixture; #[tokio::test] async fn test_snapshots_table() { diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index 94641b665d..bccf82e1fb 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -99,7 +99,8 @@ pub use runtime::{JoinHandle, Runtime, RuntimeHandle}; pub mod arrow; pub(crate) mod delete_file_index; pub mod encryption; -pub mod test_utils; +#[cfg(test)] +mod test_utils; pub mod writer; mod delete_vector; diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 7b39189bfb..7d81403969 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -652,1178 +652,39 @@ pub(crate) struct BoundPredicates { } #[cfg(test)] -pub mod tests { +mod tests { //! shared tests for the table scan API - #![allow(missing_docs)] use std::collections::HashMap; - use std::fs; - use std::fs::File; use std::sync::Arc; use arrow_array::cast::AsArray; use arrow_array::types::Int32Type; use arrow_array::{ - Array, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, RunArray, - StringArray, + Array, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RunArray, StringArray, }; use futures::{TryStreamExt, stream}; - use minijinja::value::Value; - use minijinja::{AutoEscape, Environment, context}; - use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; - use parquet::basic::Compression; - use parquet::file::properties::WriterProperties; - use tempfile::TempDir; use uuid::Uuid; use crate::arrow::ArrowReaderBuilder; use crate::expr::{BoundPredicate, Reference}; - use crate::io::{FileIO, OutputFile}; + use crate::io::FileIO; use crate::metadata_columns::{ - RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS, RESERVED_COL_NAME_FILE, RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, - RESERVED_COL_NAME_POS, RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_DELETE_FILE_PATH, - RESERVED_FIELD_ID_DELETE_FILE_POS, RESERVED_FIELD_ID_POS, + RESERVED_COL_NAME_POS, RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_POS, }; use crate::scan::{FileScanTask, FileScanTaskDeleteFile}; use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Datum, FormatVersion, Literal, - MAIN_BRANCH, ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, - MappedField, NameMapping, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, - Schema, Snapshot, SortDirection, SortField, SortOrder, Struct, StructType, Summary, - TableMetadata, TableMetadataBuilder, TableProperties, Transform, Type, - UnboundPartitionSpec, + DataContentType, DataFileFormat, Datum, Literal, MAIN_BRANCH, MappedField, NameMapping, + NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema, Snapshot, + SortDirection, SortField, SortOrder, Struct, Summary, TableMetadataBuilder, + TableProperties, Transform, Type, UnboundPartitionSpec, }; use crate::table::Table; + use crate::test_utils::scan::{TableTestFixture, assert_last_updated_seq_all}; use crate::test_utils::test_runtime; use crate::{ErrorKind, TableIdent}; - fn render_template(template: &str, ctx: Value) -> String { - let mut env = Environment::new(); - env.set_auto_escape_callback(|_| AutoEscape::None); - env.render_str(template, ctx).unwrap() - } - - /// Asserts every row of the `_last_updated_sequence_number` column across all - /// batches equals `expected` (or is null when `expected` is `None`), decoding - /// the logical value independent of the physical (run-end) encoding. - fn assert_last_updated_seq_all(batches: &[RecordBatch], expected: Option) { - use arrow_cast::cast; - use arrow_schema::DataType; - for batch in batches { - let col = batch - .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER) - .expect("_last_updated_sequence_number column should be present"); - let logical = cast(col, &DataType::Int64).unwrap(); - let values = logical.as_primitive::(); - for i in 0..values.len() { - let actual = (!values.is_null(i)).then(|| values.value(i)); - assert_eq!(actual, expected, "row {i}"); - } - } - } - - pub struct TableTestFixture { - pub table_location: String, - pub table: Table, - } - - impl TableTestFixture { - #[allow(clippy::new_without_default)] - pub fn new() -> Self { - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().join("table1"); - let manifest_list1_location = table_location.join("metadata/manifests_list_1.avro"); - let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); - let table_metadata1_location = table_location.join("metadata/v1.json"); - - let file_io = FileIO::new_with_fs(); - - let table_metadata = { - let template_json_str = fs::read_to_string(format!( - "{}/testdata/example_table_metadata_v2.json", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(); - let metadata_json = render_template(&template_json_str, context! { - table_location => &table_location, - manifest_list_1_location => &manifest_list1_location, - manifest_list_2_location => &manifest_list2_location, - table_metadata_1_location => &table_metadata1_location, - }); - serde_json::from_str::(&metadata_json).unwrap() - }; - - let table = Table::builder() - .metadata(table_metadata) - .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) - .file_io(file_io.clone()) - .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) - .runtime(test_runtime()) - .build() - .unwrap(); - - Self { - table_location: table_location.to_str().unwrap().to_string(), - table, - } - } - - #[allow(clippy::new_without_default)] - pub fn new_empty() -> Self { - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().join("table1"); - let table_metadata1_location = table_location.join("metadata/v1.json"); - - let file_io = FileIO::new_with_fs(); - - let table_metadata = { - let template_json_str = fs::read_to_string(format!( - "{}/testdata/example_empty_table_metadata_v2.json", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(); - let metadata_json = render_template(&template_json_str, context! { - table_location => &table_location, - table_metadata_1_location => &table_metadata1_location, - }); - serde_json::from_str::(&metadata_json).unwrap() - }; - - let table = Table::builder() - .metadata(table_metadata) - .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) - .file_io(file_io.clone()) - .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) - .runtime(test_runtime()) - .build() - .unwrap(); - - Self { - table_location: table_location.to_str().unwrap().to_string(), - table, - } - } - - /// Creates a fixture with 5 snapshots chained as: - /// S1 (root) -> S2 -> S3 -> S4 -> S5 (current) - /// Useful for testing snapshot history traversal. - pub fn new_with_deep_history() -> Self { - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().join("table1"); - let table_metadata1_location = table_location.join("metadata/v1.json"); - - let file_io = FileIO::new_with_fs(); - - let table_metadata = { - let json_str = fs::read_to_string(format!( - "{}/testdata/example_table_metadata_v2_deep_history.json", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(); - serde_json::from_str::(&json_str).unwrap() - }; - - let table = Table::builder() - .metadata(table_metadata) - .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) - .file_io(file_io.clone()) - .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) - .runtime(test_runtime()) - .build() - .unwrap(); - - Self { - table_location: table_location.to_str().unwrap().to_string(), - table, - } - } - - pub fn new_unpartitioned() -> Self { - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().join("table1"); - let manifest_list1_location = table_location.join("metadata/manifests_list_1.avro"); - let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); - let table_metadata1_location = table_location.join("metadata/v1.json"); - - let file_io = FileIO::new_with_fs(); - - let mut table_metadata = { - let template_json_str = fs::read_to_string(format!( - "{}/testdata/example_table_metadata_v2.json", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(); - let metadata_json = render_template(&template_json_str, context! { - table_location => &table_location, - manifest_list_1_location => &manifest_list1_location, - manifest_list_2_location => &manifest_list2_location, - table_metadata_1_location => &table_metadata1_location, - }); - serde_json::from_str::(&metadata_json).unwrap() - }; - - table_metadata.default_spec = Arc::new(PartitionSpec::unpartition_spec()); - table_metadata.partition_specs.clear(); - table_metadata.default_partition_type = StructType::new(vec![]); - table_metadata - .partition_specs - .insert(0, table_metadata.default_spec.clone()); - - let table = Table::builder() - .metadata(table_metadata) - .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) - .file_io(file_io.clone()) - .metadata_location(table_metadata1_location.to_str().unwrap()) - .runtime(test_runtime()) - .build() - .unwrap(); - - Self { - table_location: table_location.to_str().unwrap().to_string(), - table, - } - } - - pub fn new_with_partition_evolution() -> Self { - let table = Self::new().table; - let table_location = table.metadata().location.clone(); - - let manifest_list1_location = - format!("{}/metadata/manifests_list_1.avro", table_location); - let manifest_list2_location = - format!("{}/metadata/manifests_list_2.avro", table_location); - let manifest_list3_location = - format!("{}/metadata/manifests_list_3.avro", table_location); - let table_metadata1_location = format!("{}/metadata/v1.json", table_location); - - let new_table_metadata = { - let template_json_str = fs::read_to_string(format!( - "{}/testdata/example_table_metadata_v2_partition_evolution.json", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(); - let metadata_json = render_template(&template_json_str, context! { - table_location => &table_location, - manifest_list_1_location => &manifest_list1_location, - manifest_list_2_location => &manifest_list2_location, - manifest_list_3_location => &manifest_list3_location, - table_metadata_1_location => &table_metadata1_location, - }); - Arc::new(serde_json::from_str::(&metadata_json).unwrap()) - }; - - Self { - table_location, - table: table.with_metadata(new_table_metadata), - } - } - - fn next_manifest_file(&self) -> OutputFile { - self.table - .file_io() - .new_output(format!( - "{}/metadata/manifest_{}.avro", - self.table_location, - Uuid::new_v4() - )) - .unwrap() - } - - pub async fn setup_manifest_files(&mut self) { - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let parent_snapshot = current_snapshot - .parent_snapshot(self.table.metadata()) - .unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = self.table.metadata().default_partition_spec(); - - // Write the data files first, then use the file size in the manifest entries - let parquet_file_size = self.write_parquet_data_files(); - - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_data(); - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/1.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(100))])) - .key_metadata(None) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - writer - .add_delete_entry( - ManifestEntry::builder() - .status(ManifestStatus::Deleted) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/2.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(200))])) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - writer - .add_existing_entry( - ManifestEntry::builder() - .status(ManifestStatus::Existing) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/3.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(300))])) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - let data_file_manifest = writer.write_manifest_file().await.unwrap(); - - // Write to manifest list - let manifest_list_writer = self - .table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap() - .writer() - .await - .unwrap(); - let mut manifest_list_write = ManifestListWriter::v2( - manifest_list_writer, - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - ); - manifest_list_write - .add_manifests(vec![data_file_manifest].into_iter()) - .unwrap(); - manifest_list_write.close().await.unwrap(); - } - - /// Writes a v3 data manifest with a manifest-level `first_row_id` of 42, - /// so live entries inherit a per-file `first_row_id` on read. Upgrades the - /// table to v3 first, so the manifest list is read as v3. - pub async fn setup_v3_manifest_files(&mut self) { - let metadata = TableMetadataBuilder::new_from_metadata( - self.table.metadata().clone(), - self.table.metadata_location().map(str::to_string), - ) - .upgrade_format_version(FormatVersion::V3) - .unwrap() - .build() - .unwrap() - .metadata; - self.table = Table::builder() - .metadata(metadata) - .identifier(self.table.identifier().clone()) - .file_io(self.table.file_io().clone()) - .metadata_location(self.table.metadata_location().unwrap().to_string()) - .runtime(test_runtime()) - .build() - .unwrap(); - - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = self.table.metadata().default_partition_spec(); - - let parquet_file_size = self.write_parquet_data_files(); - - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v3_data(); - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/1.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(100))])) - .key_metadata(None) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - let data_file_manifest = writer.write_manifest_file().await.unwrap(); - - let manifest_list_writer = self - .table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap() - .writer() - .await - .unwrap(); - let mut manifest_list_write = ManifestListWriter::v3( - manifest_list_writer, - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - Some(42), - ); - manifest_list_write - .add_manifests(vec![data_file_manifest].into_iter()) - .unwrap(); - manifest_list_write.close().await.unwrap(); - } - - pub async fn setup_manifest_files_with_partition_evolution(&mut self) { - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let parent_snapshot = current_snapshot - .parent_snapshot(self.table.metadata()) - .unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = self.table.metadata().default_partition_spec(); - - // Write the data files first, then use the file size in the manifest entries - let parquet_file_size = self.write_parquet_data_files(); - - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_data(); - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(1) - .content(DataContentType::Data) - .file_path(format!("{}/1.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(Struct::from_iter([ - Some(Literal::long(100)), - Some(Literal::string("apa")), - Some(Literal::int(27)), - ])) - .key_metadata(None) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - writer - .add_delete_entry( - ManifestEntry::builder() - .status(ManifestStatus::Deleted) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(1) - .content(DataContentType::Data) - .file_path(format!("{}/2.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(Struct::from_iter([ - Some(Literal::long(200)), - Some(Literal::string("ice")), - Some(Literal::int(5)), - ])) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - writer - .add_existing_entry( - ManifestEntry::builder() - .status(ManifestStatus::Existing) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(1) - .content(DataContentType::Data) - .file_path(format!("{}/3.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(Struct::from_iter([ - Some(Literal::long(300)), - Some(Literal::string("apa")), - Some(Literal::int(19)), - ])) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - let data_file_manifest = writer.write_manifest_file().await.unwrap(); - - // Write to manifest list - let manifest_list_writer = self - .table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap() - .writer() - .await - .unwrap(); - let mut manifest_list_write = ManifestListWriter::v2( - manifest_list_writer, - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - ); - manifest_list_write - .add_manifests(vec![data_file_manifest].into_iter()) - .unwrap(); - manifest_list_write.close().await.unwrap(); - } - - /// Writes identical Parquet data files (1.parquet, 2.parquet, 3.parquet) - /// and returns the file size in bytes. - fn write_parquet_data_files(&self) -> u64 { - fs::create_dir_all(&self.table_location).unwrap(); - - let schema = { - let fields = vec![ - arrow_schema::Field::new("x", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - arrow_schema::Field::new("y", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - arrow_schema::Field::new("z", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - arrow_schema::Field::new("a", arrow_schema::DataType::Utf8, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "4".to_string(), - )])), - arrow_schema::Field::new("dbl", arrow_schema::DataType::Float64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "5".to_string(), - )])), - arrow_schema::Field::new("i32", arrow_schema::DataType::Int32, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "6".to_string(), - )])), - arrow_schema::Field::new("i64", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "7".to_string(), - )])), - arrow_schema::Field::new("bool", arrow_schema::DataType::Boolean, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "8".to_string(), - )])), - ]; - Arc::new(arrow_schema::Schema::new(fields)) - }; - // x: [1, 1, 1, 1, ...] - let col1 = Arc::new(Int64Array::from_iter_values(vec![1; 1024])) as ArrayRef; - - let mut values = vec![2; 512]; - values.append(vec![3; 200].as_mut()); - values.append(vec![4; 300].as_mut()); - values.append(vec![5; 12].as_mut()); - - // y: [2, 2, 2, 2, ..., 3, 3, 3, 3, ..., 4, 4, 4, 4, ..., 5, 5, 5, 5] - let col2 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - let mut values = vec![3; 512]; - values.append(vec![4; 512].as_mut()); - - // z: [3, 3, 3, 3, ..., 4, 4, 4, 4] - let col3 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - // a: ["Apache", "Apache", "Apache", ..., "Iceberg", "Iceberg", "Iceberg"] - let mut values = vec!["Apache"; 512]; - values.append(vec!["Iceberg"; 512].as_mut()); - let col4 = Arc::new(StringArray::from_iter_values(values)) as ArrayRef; - - // dbl: - let mut values = vec![100.0f64; 512]; - values.append(vec![150.0f64; 12].as_mut()); - values.append(vec![200.0f64; 500].as_mut()); - let col5 = Arc::new(Float64Array::from_iter_values(values)) as ArrayRef; - - // i32: - let mut values = vec![100i32; 512]; - values.append(vec![150i32; 12].as_mut()); - values.append(vec![200i32; 500].as_mut()); - let col6 = Arc::new(Int32Array::from_iter_values(values)) as ArrayRef; - - // i64: - let mut values = vec![100i64; 512]; - values.append(vec![150i64; 12].as_mut()); - values.append(vec![200i64; 500].as_mut()); - let col7 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - // bool: - let mut values = vec![false; 512]; - values.append(vec![true; 512].as_mut()); - let values: BooleanArray = values.into(); - let col8 = Arc::new(values) as ArrayRef; - - let to_write = RecordBatch::try_new(schema.clone(), vec![ - col1, col2, col3, col4, col5, col6, col7, col8, - ]) - .unwrap(); - - // Write the Parquet files - let props = WriterProperties::builder() - .set_compression(Compression::SNAPPY) - .build(); - - for n in 1..=3 { - let file = File::create(format!("{}/{}.parquet", &self.table_location, n)).unwrap(); - let mut writer = - ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap(); - - writer.write(&to_write).expect("Writing batch"); - - // writer must be closed to write footer - writer.close().unwrap(); - } - - fs::metadata(format!("{}/1.parquet", &self.table_location)) - .unwrap() - .len() - } - - pub async fn setup_unpartitioned_manifest_files(&mut self) { - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let parent_snapshot = current_snapshot - .parent_snapshot(self.table.metadata()) - .unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = Arc::new(PartitionSpec::unpartition_spec()); - - // Write the data files first, then use the file size in the manifest entries - let parquet_file_size = self.write_parquet_data_files(); - - // Write data files using an empty partition for unpartitioned tables. - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_data(); - - // Create an empty partition value. - let empty_partition = Struct::empty(); - - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/1.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(empty_partition.clone()) - .key_metadata(None) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - - writer - .add_delete_entry( - ManifestEntry::builder() - .status(ManifestStatus::Deleted) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/2.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(empty_partition.clone()) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - - writer - .add_existing_entry( - ManifestEntry::builder() - .status(ManifestStatus::Existing) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/3.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(empty_partition.clone()) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - - let data_file_manifest = writer.write_manifest_file().await.unwrap(); - - // Write to manifest list - let manifest_list_writer = self - .table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap() - .writer() - .await - .unwrap(); - let mut manifest_list_write = ManifestListWriter::v2( - manifest_list_writer, - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - ); - manifest_list_write - .add_manifests(vec![data_file_manifest].into_iter()) - .unwrap(); - manifest_list_write.close().await.unwrap(); - } - - pub async fn setup_deadlock_manifests(&mut self) { - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let _parent_snapshot = current_snapshot - .parent_snapshot(self.table.metadata()) - .unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = self.table.metadata().default_partition_spec(); - - // 1. Write DATA manifest with MULTIPLE entries to fill buffer - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_data(); - - // Add 10 data entries - for i in 0..10 { - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/{}.parquet", &self.table_location, i)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(100))])) - .key_metadata(None) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - } - let data_manifest = writer.write_manifest_file().await.unwrap(); - - // 2. Write DELETE manifest - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_deletes(); - - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::PositionDeletes) - .file_path(format!("{}/del.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(100))])) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - let delete_manifest = writer.write_manifest_file().await.unwrap(); - - // Write to manifest list - DATA FIRST then DELETE - // This order is crucial for reproduction - let manifest_list_writer = self - .table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap() - .writer() - .await - .unwrap(); - let mut manifest_list_write = ManifestListWriter::v2( - manifest_list_writer, - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - ); - manifest_list_write - .add_manifests(vec![data_manifest, delete_manifest].into_iter()) - .unwrap(); - manifest_list_write.close().await.unwrap(); - } - - /// Sets up a single data file `mrg.parquet` with three 100-row row groups - /// (column `x` = 1000..1300, so row position `p` carries `x = 1000 + p`) and - /// registers it in the current snapshot. When `delete_positions` is non-empty, - /// also writes a positional delete file targeting those file-absolute positions - /// and registers it in a delete manifest. - /// - /// Used to exercise the `_pos` metadata column through the real `TableScan` - /// planning path across row-group boundaries and (optionally) positional deletes. - pub async fn setup_multi_row_group_manifest(&mut self, delete_positions: &[i64]) { - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = self.table.metadata().default_partition_spec(); - - // The table's spec 0 is identity on `x`, so give the data and delete files a - // fixed partition value. Filter tests deliberately filter on `y` (a - // non-partition column) so pruning is driven by Parquet row-group statistics - // rather than partition values. - let partition = Struct::from_iter([Some(Literal::long(1000))]); - - let (data_file_path, data_file_size) = self.write_multi_row_group_data_file(); - - let mut data_writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_data(); - data_writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(data_file_path.clone()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(data_file_size) - .record_count(300) - .partition(partition.clone()) - .key_metadata(None) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - let data_manifest = data_writer.write_manifest_file().await.unwrap(); - - let mut manifests = vec![data_manifest]; - - if !delete_positions.is_empty() { - let (del_path, del_size) = - self.write_positional_delete_file(&data_file_path, delete_positions); - - let mut delete_writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_deletes(); - delete_writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::PositionDeletes) - .file_path(del_path) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(del_size) - .record_count(delete_positions.len() as u64) - .partition(partition.clone()) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - manifests.push(delete_writer.write_manifest_file().await.unwrap()); - } - - let manifest_list_writer = self - .table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap() - .writer() - .await - .unwrap(); - let mut manifest_list_write = ManifestListWriter::v2( - manifest_list_writer, - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - ); - manifest_list_write - .add_manifests(manifests.into_iter()) - .unwrap(); - manifest_list_write.close().await.unwrap(); - } - - /// Writes a manifest with three live "Added" data-file entries (partitioned on `x` - /// = 100, 200, 300), each with the given `sort_order_id` set on its `DataFile` - /// (`None` leaves the field unset). Used to test how `sort_order_id` resolution - /// against the table's sort orders flows into each entry's `FileScanTask`. - pub async fn setup_manifest_files_with_sort_order_ids( - &mut self, - sort_order_ids: [Option; 4], - ) { - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = self.table.metadata().default_partition_spec(); - let parquet_file_size = self.write_parquet_data_files(); - - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_data(); - - for (i, sort_order_id) in sort_order_ids.into_iter().enumerate() { - let mut data_file_builder = DataFileBuilder::default(); - data_file_builder - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/{}.parquet", &self.table_location, i + 1)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(parquet_file_size) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long( - 100 * (i as i64 + 1), - ))])); - if let Some(id) = sort_order_id { - data_file_builder.sort_order_id(id); - } - let data_file = data_file_builder.build().unwrap(); - - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file(data_file) - .build(), - ) - .unwrap(); - } - - let data_file_manifest = writer.write_manifest_file().await.unwrap(); - - let manifest_list_writer = self - .table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap() - .writer() - .await - .unwrap(); - let mut manifest_list_write = ManifestListWriter::v2( - manifest_list_writer, - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - ); - manifest_list_write - .add_manifests(std::iter::once(data_file_manifest)) - .unwrap(); - manifest_list_write.close().await.unwrap(); - } - - /// Writes `mrg.parquet` with three 100-row row groups. Columns `x` (field - /// id `1`) and `y` (field id `2`) both run 1000..1300, so row position `p` - /// carries `x = y = 1000 + p`. Returns `(path, file_size_in_bytes)`. - fn write_multi_row_group_data_file(&self) -> (String, u64) { - fs::create_dir_all(&self.table_location).unwrap(); - - let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![ - arrow_schema::Field::new("x", arrow_schema::DataType::Int64, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]), - ), - arrow_schema::Field::new("y", arrow_schema::DataType::Int64, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())]), - ), - ])); - - let path = format!("{}/mrg.parquet", &self.table_location); - let max_row_group_row_count = 100; - let props = WriterProperties::builder() - .set_compression(Compression::SNAPPY) - .set_max_row_group_row_count(Some(max_row_group_row_count)) - .build(); - - let file = File::create(&path).unwrap(); - let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); - for group in 0..3i64 { - let base = 1000 + group * max_row_group_row_count as i64; - let col = Arc::new(Int64Array::from_iter_values( - base..base + max_row_group_row_count as i64, - )) as ArrayRef; - let batch = - RecordBatch::try_new(arrow_schema.clone(), vec![col.clone(), col]).unwrap(); - writer.write(&batch).unwrap(); - } - writer.close().unwrap(); - - let size = fs::metadata(&path).unwrap().len(); - (path, size) - } - - /// Writes a positional delete file targeting `positions` in `data_path`. - /// Returns `(path, file_size_in_bytes)`. - fn write_positional_delete_file( - &self, - data_path: &str, - positions: &[i64], - ) -> (String, u64) { - let del_schema = Arc::new(arrow_schema::Schema::new(vec![ - arrow_schema::Field::new( - RESERVED_COL_NAME_DELETE_FILE_PATH, - arrow_schema::DataType::Utf8, - false, - ) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - RESERVED_FIELD_ID_DELETE_FILE_PATH.to_string(), // 2147483546 - )])), - arrow_schema::Field::new( - RESERVED_COL_NAME_DELETE_FILE_POS, - arrow_schema::DataType::Int64, - false, - ) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - RESERVED_FIELD_ID_DELETE_FILE_POS.to_string(), // 2147483545 - )])), - ])); - - let batch = RecordBatch::try_new(del_schema.clone(), vec![ - Arc::new(StringArray::from_iter_values(std::iter::repeat_n( - data_path.to_string(), - positions.len(), - ))) as ArrayRef, - Arc::new(Int64Array::from_iter_values(positions.iter().copied())) as ArrayRef, - ]) - .unwrap(); - - let path = format!("{}/pos-del.parquet", &self.table_location); - let props = WriterProperties::builder() - .set_compression(Compression::SNAPPY) - .build(); - let file = File::create(&path).unwrap(); - let mut writer = ArrowWriter::try_new(file, del_schema, Some(props)).unwrap(); - writer.write(&batch).unwrap(); - writer.close().unwrap(); - - let size = fs::metadata(&path).unwrap().len(); - (path, size) - } - } - #[tokio::test] async fn test_table_scan_columns() { let table = TableTestFixture::new().table; diff --git a/crates/iceberg/src/test_utils.rs b/crates/iceberg/src/test_utils.rs deleted file mode 100644 index e80a51d98d..0000000000 --- a/crates/iceberg/src/test_utils.rs +++ /dev/null @@ -1,190 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Test utilities. -//! This module is pub just for internal testing. -//! It is subject to change and is not intended to be used by external users. - -use std::sync::{Arc, OnceLock}; - -use arrow_array::RecordBatch; -use expect_test::Expect; -use itertools::Itertools; -#[cfg(test)] -use roaring::RoaringTreemap; - -use crate::TableIdent; -#[cfg(test)] -use crate::encryption::EncryptionManager; -use crate::encryption::SensitiveBytes; -use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; -use crate::io::FileIO; -use crate::runtime::Runtime; -use crate::spec::TableMetadata; -use crate::table::Table; - -/// Returns a process-wide [`Runtime`] suitable for tests that need to construct -/// a [`Table`] outside a tokio context. -/// -/// The returned [`Runtime`] wraps a single shared multi-thread tokio runtime -/// that is lazily built on first call and lives until process exit. Cloning is -/// cheap, so test code can call this every time it needs a runtime to feed -/// into [`TableBuilder::runtime`](crate::table::TableBuilder::runtime). -pub fn test_runtime() -> Runtime { - static TOKIO_RT: OnceLock = OnceLock::new(); - let tokio_rt = TOKIO_RT.get_or_init(|| { - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .expect("failed to build test tokio runtime") - }); - Runtime::new(tokio_rt) -} - -/// Snapshot testing to check the resulting record batch. -/// -/// - `expected_schema/data`: put `expect![[""]]` as a placeholder, -/// and then run test with `UPDATE_EXPECT=1 cargo test` to automatically update the result, -/// or use rust-analyzer (see [video](https://github.com/rust-analyzer/expect-test)). -/// Check the doc of [`expect_test`] for more details. -/// - `ignore_check_columns`: Some columns are not stable, so we can skip them. -/// - `sort_column`: The order of the data might be non-deterministic, so we can sort it by a column. -pub fn check_record_batches( - record_batches: Vec, - expected_schema: Expect, - expected_data: Expect, - ignore_check_columns: &[&str], - sort_column: Option<&str>, -) { - assert!(!record_batches.is_empty(), "Empty record batches"); - - // Combine record batches using the first batch's schema - let first_batch = record_batches.first().unwrap(); - let record_batch = - arrow_select::concat::concat_batches(&first_batch.schema(), &record_batches).unwrap(); - - let mut columns = record_batch.columns().to_vec(); - if let Some(sort_column) = sort_column { - let column = record_batch.column_by_name(sort_column).unwrap(); - let indices = arrow_ord::sort::sort_to_indices(column, None, None).unwrap(); - columns = columns - .iter() - .map(|column| arrow_select::take::take(column.as_ref(), &indices, None).unwrap()) - .collect_vec(); - } - - expected_schema.assert_eq(&format!( - "{}", - record_batch.schema().fields().iter().format(",\n") - )); - expected_data.assert_eq(&format!( - "{}", - record_batch - .schema() - .fields() - .iter() - .zip_eq(columns) - .map(|(field, column)| { - if ignore_check_columns.contains(&field.name().as_str()) { - format!("{}: (skipped)", field.name()) - } else { - format!("{}: {:?}", field.name(), column) - } - }) - .format(",\n") - )); -} - -/// An [`EncryptionManager`] backed by an in-memory KMS holding `table_key_id`. -#[cfg(test)] -pub(crate) fn make_encryption_manager(table_key_id: &str) -> Arc { - let kms = MemoryKeyManagementClient::new(); - kms.add_master_key(table_key_id).unwrap(); - Arc::new( - EncryptionManager::builder() - .kms_client(Arc::new(kms) as Arc) - .table_key_id(table_key_id) - .build(), - ) -} - -/// Encodes a `deletion-vector-v1` Puffin blob for the given positions, matching the framing in -/// [`DeleteVector::deserialize`](crate::delete_vector::DeleteVector::deserialize). -#[cfg(test)] -pub(crate) fn encode_dv_blob(positions: impl IntoIterator) -> Vec { - let mut bitmap = RoaringTreemap::new(); - for pos in positions { - bitmap.insert(pos); - } - let mut vector = Vec::new(); - bitmap.serialize_into(&mut vector).unwrap(); - crate::delete_vector::frame_dv_blob(&vector) -} - -/// Build a table backed by the V3 encryption fixture and an in-memory KMS, -/// so it has an [`EncryptionManager`](crate::encryption::EncryptionManager). -/// -/// The fixture's snapshot references an encrypted manifest list; its bytes -/// (the `manifest-list-v3-encrypted.avro` testdata, an encrypted empty list) -/// are seeded into the in-memory `FileIO` at that path so callers can read -/// the current snapshot's manifest list. -pub async fn make_encrypted_table() -> Table { - let metadata_json = std::fs::read_to_string(format!( - "{}/testdata/table_metadata/TableMetadataV3ValidEncryption.json", - env!("CARGO_MANIFEST_DIR"), - )) - .unwrap(); - let metadata: TableMetadata = serde_json::from_str(&metadata_json).unwrap(); - - let kms: Arc = { - let k = MemoryKeyManagementClient::new(); - k.add_master_key_bytes( - "master-1", - SensitiveBytes::new([ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, - 0x0e, 0x0f, - ]), - ) - .unwrap(); - Arc::new(k) - }; - - let file_io = FileIO::new_with_memory(); - - // Seed the encrypted (empty) manifest list at the path the snapshot references. - let manifest_list_bytes = std::fs::read(format!( - "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro", - env!("CARGO_MANIFEST_DIR"), - )) - .unwrap(); - file_io - .new_output(metadata.current_snapshot().unwrap().manifest_list()) - .unwrap() - .write(manifest_list_bytes.into()) - .await - .unwrap(); - - Table::builder() - .metadata(metadata) - .metadata_location("memory:///table/metadata/v1.json") - .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(file_io) - .kms_client(kms) - .runtime(test_runtime()) - .build() - .unwrap() -} diff --git a/crates/iceberg/src/test_utils/delete_vector.rs b/crates/iceberg/src/test_utils/delete_vector.rs new file mode 100644 index 0000000000..b06bd74983 --- /dev/null +++ b/crates/iceberg/src/test_utils/delete_vector.rs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Fixtures for tests that need encoded deletion vectors. + +use roaring::RoaringTreemap; + +/// Encodes a `deletion-vector-v1` Puffin blob for the given positions, matching the framing in +/// [`DeleteVector::deserialize`](crate::delete_vector::DeleteVector::deserialize). +pub(crate) fn encode_dv_blob(positions: impl IntoIterator) -> Vec { + let mut bitmap = RoaringTreemap::new(); + for pos in positions { + bitmap.insert(pos); + } + let mut vector = Vec::new(); + bitmap.serialize_into(&mut vector).unwrap(); + crate::delete_vector::frame_dv_blob(&vector) +} diff --git a/crates/iceberg/src/test_utils/encryption.rs b/crates/iceberg/src/test_utils/encryption.rs new file mode 100644 index 0000000000..2accd5255c --- /dev/null +++ b/crates/iceberg/src/test_utils/encryption.rs @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Fixtures for tests that need an [`EncryptionManager`] or an encrypted table. + +use std::sync::Arc; + +use super::test_runtime; +use crate::TableIdent; +use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; +use crate::encryption::{EncryptionManager, SensitiveBytes}; +use crate::io::FileIO; +use crate::spec::TableMetadata; +use crate::table::Table; + +/// An [`EncryptionManager`] backed by an in-memory KMS holding `table_key_id`. +pub(crate) fn make_encryption_manager(table_key_id: &str) -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key(table_key_id).unwrap(); + Arc::new( + EncryptionManager::builder() + .kms_client(Arc::new(kms) as Arc) + .table_key_id(table_key_id) + .build(), + ) +} + +/// Build a table backed by the V3 encryption fixture and an in-memory KMS, +/// so it has an [`EncryptionManager`](crate::encryption::EncryptionManager). +/// +/// The fixture's snapshot references an encrypted manifest list; its bytes +/// (the `manifest-list-v3-encrypted.avro` testdata, an encrypted empty list) +/// are seeded into the in-memory `FileIO` at that path so callers can read +/// the current snapshot's manifest list. +pub(crate) async fn make_encrypted_table() -> Table { + let metadata_json = std::fs::read_to_string(format!( + "{}/testdata/table_metadata/TableMetadataV3ValidEncryption.json", + env!("CARGO_MANIFEST_DIR"), + )) + .unwrap(); + let metadata: TableMetadata = serde_json::from_str(&metadata_json).unwrap(); + + let kms: Arc = { + let k = MemoryKeyManagementClient::new(); + k.add_master_key_bytes( + "master-1", + SensitiveBytes::new([ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, + 0x0e, 0x0f, + ]), + ) + .unwrap(); + Arc::new(k) + }; + + let file_io = FileIO::new_with_memory(); + + // Seed the encrypted (empty) manifest list at the path the snapshot references. + let manifest_list_bytes = std::fs::read(format!( + "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro", + env!("CARGO_MANIFEST_DIR"), + )) + .unwrap(); + file_io + .new_output(metadata.current_snapshot().unwrap().manifest_list()) + .unwrap() + .write(manifest_list_bytes.into()) + .await + .unwrap(); + + Table::builder() + .metadata(metadata) + .metadata_location("memory:///table/metadata/v1.json") + .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) + .file_io(file_io) + .kms_client(kms) + .runtime(test_runtime()) + .build() + .unwrap() +} diff --git a/crates/iceberg/src/test_utils/mod.rs b/crates/iceberg/src/test_utils/mod.rs new file mode 100644 index 0000000000..753a17e533 --- /dev/null +++ b/crates/iceberg/src/test_utils/mod.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Test utilities for this crate's own tests. +//! +//! Compiled under `cfg(test)` only, so nothing here is public API. Fixtures +//! that other crates in the workspace need from their own tests live in the +//! `iceberg_test_utils` crate instead. + +mod delete_vector; +mod encryption; +mod record_batch; +mod runtime; +pub(crate) mod scan; + +pub(crate) use delete_vector::encode_dv_blob; +pub(crate) use encryption::{make_encrypted_table, make_encryption_manager}; +pub(crate) use record_batch::check_record_batches; +pub(crate) use runtime::test_runtime; diff --git a/crates/iceberg/src/test_utils/record_batch.rs b/crates/iceberg/src/test_utils/record_batch.rs new file mode 100644 index 0000000000..7e37f260df --- /dev/null +++ b/crates/iceberg/src/test_utils/record_batch.rs @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Snapshot assertions over [`RecordBatch`] results. + +use arrow_array::RecordBatch; +use expect_test::Expect; +use itertools::Itertools; + +/// Snapshot testing to check the resulting record batch. +/// +/// - `expected_schema/data`: put `expect![[""]]` as a placeholder, +/// and then run test with `UPDATE_EXPECT=1 cargo test` to automatically update the result, +/// or use rust-analyzer (see [video](https://github.com/rust-analyzer/expect-test)). +/// Check the doc of [`expect_test`] for more details. +/// - `ignore_check_columns`: Some columns are not stable, so we can skip them. +/// - `sort_column`: The order of the data might be non-deterministic, so we can sort it by a column. +pub(crate) fn check_record_batches( + record_batches: Vec, + expected_schema: Expect, + expected_data: Expect, + ignore_check_columns: &[&str], + sort_column: Option<&str>, +) { + assert!(!record_batches.is_empty(), "Empty record batches"); + + // Combine record batches using the first batch's schema + let first_batch = record_batches.first().unwrap(); + let record_batch = + arrow_select::concat::concat_batches(&first_batch.schema(), &record_batches).unwrap(); + + let mut columns = record_batch.columns().to_vec(); + if let Some(sort_column) = sort_column { + let column = record_batch.column_by_name(sort_column).unwrap(); + let indices = arrow_ord::sort::sort_to_indices(column, None, None).unwrap(); + columns = columns + .iter() + .map(|column| arrow_select::take::take(column.as_ref(), &indices, None).unwrap()) + .collect_vec(); + } + + expected_schema.assert_eq(&format!( + "{}", + record_batch.schema().fields().iter().format(",\n") + )); + expected_data.assert_eq(&format!( + "{}", + record_batch + .schema() + .fields() + .iter() + .zip_eq(columns) + .map(|(field, column)| { + if ignore_check_columns.contains(&field.name().as_str()) { + format!("{}: (skipped)", field.name()) + } else { + format!("{}: {:?}", field.name(), column) + } + }) + .format(",\n") + )); +} diff --git a/crates/iceberg/src/test_utils/runtime.rs b/crates/iceberg/src/test_utils/runtime.rs new file mode 100644 index 0000000000..2070e6cb26 --- /dev/null +++ b/crates/iceberg/src/test_utils/runtime.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A shared tokio runtime for tests that build a [`Table`](crate::table::Table). +//! +//! Twin of `iceberg_test_utils::test_runtime`, which serves other crates in the +//! workspace. The two cannot be merged: a dev-dependency links a separate +//! instance of this crate, so a `Runtime` built there is a distinct type from +//! `crate::Runtime` here. + +use std::sync::OnceLock; + +use crate::runtime::Runtime; + +/// Returns a process-wide [`Runtime`] suitable for tests that need to construct +/// a [`Table`](crate::table::Table) outside a tokio context. +/// +/// The returned [`Runtime`] wraps a single shared multi-thread tokio runtime +/// that is lazily built on first call and lives until process exit. Cloning is +/// cheap, so test code can call this every time it needs a runtime to feed +/// into [`TableBuilder::runtime`](crate::table::TableBuilder::runtime). +pub(crate) fn test_runtime() -> Runtime { + static TOKIO_RT: OnceLock = OnceLock::new(); + let tokio_rt = TOKIO_RT.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build test tokio runtime") + }); + Runtime::new(tokio_rt) +} diff --git a/crates/iceberg/src/test_utils/scan.rs b/crates/iceberg/src/test_utils/scan.rs new file mode 100644 index 0000000000..0c4a7e9dd7 --- /dev/null +++ b/crates/iceberg/src/test_utils/scan.rs @@ -0,0 +1,1160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared test fixtures for the table scan API. + +use std::collections::HashMap; +use std::fs; +use std::fs::File; +use std::sync::Arc; + +use arrow_array::cast::AsArray; +use arrow_array::{ + Array, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray, +}; +use minijinja::value::Value; +use minijinja::{AutoEscape, Environment, context}; +use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; +use tempfile::TempDir; +use uuid::Uuid; + +use crate::TableIdent; +use crate::io::{FileIO, OutputFile}; +use crate::metadata_columns::{ + RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS, + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_FIELD_ID_DELETE_FILE_PATH, + RESERVED_FIELD_ID_DELETE_FILE_POS, +}; +use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, FormatVersion, Literal, ManifestEntry, + ManifestListWriter, ManifestStatus, ManifestWriterBuilder, PartitionSpec, Struct, StructType, + TableMetadata, TableMetadataBuilder, +}; +use crate::table::Table; +use crate::test_utils::test_runtime; + +fn render_template(template: &str, ctx: Value) -> String { + let mut env = Environment::new(); + env.set_auto_escape_callback(|_| AutoEscape::None); + env.render_str(template, ctx).unwrap() +} + +/// Asserts every row of the `_last_updated_sequence_number` column across all +/// batches equals `expected` (or is null when `expected` is `None`), decoding +/// the logical value independent of the physical (run-end) encoding. +pub(crate) fn assert_last_updated_seq_all(batches: &[RecordBatch], expected: Option) { + use arrow_cast::cast; + use arrow_schema::DataType; + for batch in batches { + let col = batch + .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER) + .expect("_last_updated_sequence_number column should be present"); + let logical = cast(col, &DataType::Int64).unwrap(); + let values = logical.as_primitive::(); + for i in 0..values.len() { + let actual = (!values.is_null(i)).then(|| values.value(i)); + assert_eq!(actual, expected, "row {i}"); + } + } +} + +pub(crate) struct TableTestFixture { + pub(crate) table_location: String, + pub(crate) table: Table, +} + +impl TableTestFixture { + #[allow(clippy::new_without_default)] + pub(crate) fn new() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let manifest_list1_location = table_location.join("metadata/manifests_list_1.avro"); + let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let table_metadata = { + let template_json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + manifest_list_1_location => &manifest_list1_location, + manifest_list_2_location => &manifest_list2_location, + table_metadata_1_location => &table_metadata1_location, + }); + serde_json::from_str::(&metadata_json).unwrap() + }; + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + + #[allow(clippy::new_without_default)] + pub(crate) fn new_empty() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let table_metadata = { + let template_json_str = fs::read_to_string(format!( + "{}/testdata/example_empty_table_metadata_v2.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + table_metadata_1_location => &table_metadata1_location, + }); + serde_json::from_str::(&metadata_json).unwrap() + }; + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + + /// Creates a fixture with 5 snapshots chained as: + /// S1 (root) -> S2 -> S3 -> S4 -> S5 (current) + /// Useful for testing snapshot history traversal. + pub(crate) fn new_with_deep_history() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let table_metadata = { + let json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2_deep_history.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + serde_json::from_str::(&json_str).unwrap() + }; + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + + pub(crate) fn new_unpartitioned() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let manifest_list1_location = table_location.join("metadata/manifests_list_1.avro"); + let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let mut table_metadata = { + let template_json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + manifest_list_1_location => &manifest_list1_location, + manifest_list_2_location => &manifest_list2_location, + table_metadata_1_location => &table_metadata1_location, + }); + serde_json::from_str::(&metadata_json).unwrap() + }; + + table_metadata.default_spec = Arc::new(PartitionSpec::unpartition_spec()); + table_metadata.partition_specs.clear(); + table_metadata.default_partition_type = StructType::new(vec![]); + table_metadata + .partition_specs + .insert(0, table_metadata.default_spec.clone()); + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + + pub(crate) fn new_with_partition_evolution() -> Self { + let table = Self::new().table; + let table_location = table.metadata().location.clone(); + + let manifest_list1_location = format!("{}/metadata/manifests_list_1.avro", table_location); + let manifest_list2_location = format!("{}/metadata/manifests_list_2.avro", table_location); + let manifest_list3_location = format!("{}/metadata/manifests_list_3.avro", table_location); + let table_metadata1_location = format!("{}/metadata/v1.json", table_location); + + let new_table_metadata = { + let template_json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2_partition_evolution.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + manifest_list_1_location => &manifest_list1_location, + manifest_list_2_location => &manifest_list2_location, + manifest_list_3_location => &manifest_list3_location, + table_metadata_1_location => &table_metadata1_location, + }); + Arc::new(serde_json::from_str::(&metadata_json).unwrap()) + }; + + Self { + table_location, + table: table.with_metadata(new_table_metadata), + } + } + + fn next_manifest_file(&self) -> OutputFile { + self.table + .file_io() + .new_output(format!( + "{}/metadata/manifest_{}.avro", + self.table_location, + Uuid::new_v4() + )) + .unwrap() + } + + pub(crate) async fn setup_manifest_files(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // Write the data files first, then use the file size in the manifest entries + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + writer + .add_delete_entry( + ManifestEntry::builder() + .status(ManifestStatus::Deleted) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/2.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(200))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + writer + .add_existing_entry( + ManifestEntry::builder() + .status(ManifestStatus::Existing) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/3.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes a v3 data manifest with a manifest-level `first_row_id` of 42, + /// so live entries inherit a per-file `first_row_id` on read. Upgrades the + /// table to v3 first, so the manifest list is read as v3. + pub(crate) async fn setup_v3_manifest_files(&mut self) { + let metadata = TableMetadataBuilder::new_from_metadata( + self.table.metadata().clone(), + self.table.metadata_location().map(str::to_string), + ) + .upgrade_format_version(FormatVersion::V3) + .unwrap() + .build() + .unwrap() + .metadata; + self.table = Table::builder() + .metadata(metadata) + .identifier(self.table.identifier().clone()) + .file_io(self.table.file_io().clone()) + .metadata_location(self.table.metadata_location().unwrap().to_string()) + .runtime(test_runtime()) + .build() + .unwrap(); + + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v3_data(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v3( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + Some(42), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + pub(crate) async fn setup_manifest_files_with_partition_evolution(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // Write the data files first, then use the file size in the manifest entries + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(1) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([ + Some(Literal::long(100)), + Some(Literal::string("apa")), + Some(Literal::int(27)), + ])) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + writer + .add_delete_entry( + ManifestEntry::builder() + .status(ManifestStatus::Deleted) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(1) + .content(DataContentType::Data) + .file_path(format!("{}/2.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([ + Some(Literal::long(200)), + Some(Literal::string("ice")), + Some(Literal::int(5)), + ])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + writer + .add_existing_entry( + ManifestEntry::builder() + .status(ManifestStatus::Existing) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(1) + .content(DataContentType::Data) + .file_path(format!("{}/3.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([ + Some(Literal::long(300)), + Some(Literal::string("apa")), + Some(Literal::int(19)), + ])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes identical Parquet data files (1.parquet, 2.parquet, 3.parquet) + /// and returns the file size in bytes. + fn write_parquet_data_files(&self) -> u64 { + fs::create_dir_all(&self.table_location).unwrap(); + + let schema = { + let fields = vec![ + arrow_schema::Field::new("x", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]), + ), + arrow_schema::Field::new("y", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())]), + ), + arrow_schema::Field::new("z", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())]), + ), + arrow_schema::Field::new("a", arrow_schema::DataType::Utf8, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "4".to_string())]), + ), + arrow_schema::Field::new("dbl", arrow_schema::DataType::Float64, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "5".to_string(), + )])), + arrow_schema::Field::new("i32", arrow_schema::DataType::Int32, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "6".to_string(), + )])), + arrow_schema::Field::new("i64", arrow_schema::DataType::Int64, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "7".to_string(), + )])), + arrow_schema::Field::new("bool", arrow_schema::DataType::Boolean, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "8".to_string(), + )])), + ]; + Arc::new(arrow_schema::Schema::new(fields)) + }; + // x: [1, 1, 1, 1, ...] + let col1 = Arc::new(Int64Array::from_iter_values(vec![1; 1024])) as ArrayRef; + + let mut values = vec![2; 512]; + values.append(vec![3; 200].as_mut()); + values.append(vec![4; 300].as_mut()); + values.append(vec![5; 12].as_mut()); + + // y: [2, 2, 2, 2, ..., 3, 3, 3, 3, ..., 4, 4, 4, 4, ..., 5, 5, 5, 5] + let col2 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + + let mut values = vec![3; 512]; + values.append(vec![4; 512].as_mut()); + + // z: [3, 3, 3, 3, ..., 4, 4, 4, 4] + let col3 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + + // a: ["Apache", "Apache", "Apache", ..., "Iceberg", "Iceberg", "Iceberg"] + let mut values = vec!["Apache"; 512]; + values.append(vec!["Iceberg"; 512].as_mut()); + let col4 = Arc::new(StringArray::from_iter_values(values)) as ArrayRef; + + // dbl: + let mut values = vec![100.0f64; 512]; + values.append(vec![150.0f64; 12].as_mut()); + values.append(vec![200.0f64; 500].as_mut()); + let col5 = Arc::new(Float64Array::from_iter_values(values)) as ArrayRef; + + // i32: + let mut values = vec![100i32; 512]; + values.append(vec![150i32; 12].as_mut()); + values.append(vec![200i32; 500].as_mut()); + let col6 = Arc::new(Int32Array::from_iter_values(values)) as ArrayRef; + + // i64: + let mut values = vec![100i64; 512]; + values.append(vec![150i64; 12].as_mut()); + values.append(vec![200i64; 500].as_mut()); + let col7 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + + // bool: + let mut values = vec![false; 512]; + values.append(vec![true; 512].as_mut()); + let values: BooleanArray = values.into(); + let col8 = Arc::new(values) as ArrayRef; + + let to_write = RecordBatch::try_new(schema.clone(), vec![ + col1, col2, col3, col4, col5, col6, col7, col8, + ]) + .unwrap(); + + // Write the Parquet files + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + for n in 1..=3 { + let file = File::create(format!("{}/{}.parquet", &self.table_location, n)).unwrap(); + let mut writer = + ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap(); + + writer.write(&to_write).expect("Writing batch"); + + // writer must be closed to write footer + writer.close().unwrap(); + } + + fs::metadata(format!("{}/1.parquet", &self.table_location)) + .unwrap() + .len() + } + + pub(crate) async fn setup_unpartitioned_manifest_files(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = Arc::new(PartitionSpec::unpartition_spec()); + + // Write the data files first, then use the file size in the manifest entries + let parquet_file_size = self.write_parquet_data_files(); + + // Write data files using an empty partition for unpartitioned tables. + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + + // Create an empty partition value. + let empty_partition = Struct::empty(); + + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(empty_partition.clone()) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + + writer + .add_delete_entry( + ManifestEntry::builder() + .status(ManifestStatus::Deleted) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/2.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(empty_partition.clone()) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + + writer + .add_existing_entry( + ManifestEntry::builder() + .status(ManifestStatus::Existing) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/3.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(empty_partition.clone()) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + pub(crate) async fn setup_deadlock_manifests(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let _parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // 1. Write DATA manifest with MULTIPLE entries to fill buffer + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + + // Add 10 data entries + for i in 0..10 { + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/{}.parquet", &self.table_location, i)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + } + let data_manifest = writer.write_manifest_file().await.unwrap(); + + // 2. Write DELETE manifest + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_deletes(); + + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::PositionDeletes) + .file_path(format!("{}/del.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let delete_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list - DATA FIRST then DELETE + // This order is crucial for reproduction + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_manifest, delete_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Sets up a single data file `mrg.parquet` with three 100-row row groups + /// (column `x` = 1000..1300, so row position `p` carries `x = 1000 + p`) and + /// registers it in the current snapshot. When `delete_positions` is non-empty, + /// also writes a positional delete file targeting those file-absolute positions + /// and registers it in a delete manifest. + /// + /// Used to exercise the `_pos` metadata column through the real `TableScan` + /// planning path across row-group boundaries and (optionally) positional deletes. + pub(crate) async fn setup_multi_row_group_manifest(&mut self, delete_positions: &[i64]) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // The table's spec 0 is identity on `x`, so give the data and delete files a + // fixed partition value. Filter tests deliberately filter on `y` (a + // non-partition column) so pruning is driven by Parquet row-group statistics + // rather than partition values. + let partition = Struct::from_iter([Some(Literal::long(1000))]); + + let (data_file_path, data_file_size) = self.write_multi_row_group_data_file(); + + let mut data_writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + data_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(data_file_path.clone()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(data_file_size) + .record_count(300) + .partition(partition.clone()) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_manifest = data_writer.write_manifest_file().await.unwrap(); + + let mut manifests = vec![data_manifest]; + + if !delete_positions.is_empty() { + let (del_path, del_size) = + self.write_positional_delete_file(&data_file_path, delete_positions); + + let mut delete_writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_deletes(); + delete_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::PositionDeletes) + .file_path(del_path) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(del_size) + .record_count(delete_positions.len() as u64) + .partition(partition.clone()) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + manifests.push(delete_writer.write_manifest_file().await.unwrap()); + } + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(manifests.into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes a manifest with three live "Added" data-file entries (partitioned on `x` + /// = 100, 200, 300), each with the given `sort_order_id` set on its `DataFile` + /// (`None` leaves the field unset). Used to test how `sort_order_id` resolution + /// against the table's sort orders flows into each entry's `FileScanTask`. + pub(crate) async fn setup_manifest_files_with_sort_order_ids( + &mut self, + sort_order_ids: [Option; 4], + ) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + + for (i, sort_order_id) in sort_order_ids.into_iter().enumerate() { + let mut data_file_builder = DataFileBuilder::default(); + data_file_builder + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/{}.parquet", &self.table_location, i + 1)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long( + 100 * (i as i64 + 1), + ))])); + if let Some(id) = sort_order_id { + data_file_builder.sort_order_id(id); + } + let data_file = data_file_builder.build().unwrap(); + + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(data_file) + .build(), + ) + .unwrap(); + } + + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(std::iter::once(data_file_manifest)) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes `mrg.parquet` with three 100-row row groups. Columns `x` (field + /// id `1`) and `y` (field id `2`) both run 1000..1300, so row position `p` + /// carries `x = y = 1000 + p`. Returns `(path, file_size_in_bytes)`. + fn write_multi_row_group_data_file(&self) -> (String, u64) { + fs::create_dir_all(&self.table_location).unwrap(); + + let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![ + arrow_schema::Field::new("x", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]), + ), + arrow_schema::Field::new("y", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())]), + ), + ])); + + let path = format!("{}/mrg.parquet", &self.table_location); + let max_row_group_row_count = 100; + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(max_row_group_row_count)) + .build(); + + let file = File::create(&path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + for group in 0..3i64 { + let base = 1000 + group * max_row_group_row_count as i64; + let col = Arc::new(Int64Array::from_iter_values( + base..base + max_row_group_row_count as i64, + )) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![col.clone(), col]).unwrap(); + writer.write(&batch).unwrap(); + } + writer.close().unwrap(); + + let size = fs::metadata(&path).unwrap().len(); + (path, size) + } + + /// Writes a positional delete file targeting `positions` in `data_path`. + /// Returns `(path, file_size_in_bytes)`. + fn write_positional_delete_file(&self, data_path: &str, positions: &[i64]) -> (String, u64) { + let del_schema = Arc::new(arrow_schema::Schema::new(vec![ + arrow_schema::Field::new( + RESERVED_COL_NAME_DELETE_FILE_PATH, + arrow_schema::DataType::Utf8, + false, + ) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_DELETE_FILE_PATH.to_string(), // 2147483546 + )])), + arrow_schema::Field::new( + RESERVED_COL_NAME_DELETE_FILE_POS, + arrow_schema::DataType::Int64, + false, + ) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_DELETE_FILE_POS.to_string(), // 2147483545 + )])), + ])); + + let batch = RecordBatch::try_new(del_schema.clone(), vec![ + Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + data_path.to_string(), + positions.len(), + ))) as ArrayRef, + Arc::new(Int64Array::from_iter_values(positions.iter().copied())) as ArrayRef, + ]) + .unwrap(); + + let path = format!("{}/pos-del.parquet", &self.table_location); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&path).unwrap(); + let mut writer = ArrowWriter::try_new(file, del_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let size = fs::metadata(&path).unwrap().len(); + (path, size) + } +} diff --git a/crates/iceberg/src/util/snapshot.rs b/crates/iceberg/src/util/snapshot.rs index d5b87a340d..deac53ef84 100644 --- a/crates/iceberg/src/util/snapshot.rs +++ b/crates/iceberg/src/util/snapshot.rs @@ -79,7 +79,7 @@ pub fn ancestors_between( #[cfg(test)] mod tests { use super::*; - use crate::scan::tests::TableTestFixture; + use crate::test_utils::scan::TableTestFixture; // Five snapshots chained as: S1 (root) -> S2 -> S3 -> S4 -> S5 (current) const S1: i64 = 3051729675574597004; diff --git a/crates/test_utils/Cargo.toml b/crates/test_utils/Cargo.toml index eae0b32a4c..41f4e38dd8 100644 --- a/crates/test_utils/Cargo.toml +++ b/crates/test_utils/Cargo.toml @@ -28,6 +28,7 @@ repository = { workspace = true } [dependencies] iceberg = { workspace = true } +tokio = { workspace = true } tracing-subscriber = { workspace = true } [features] diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index b64e462696..bf2dad0ed6 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -19,8 +19,13 @@ //! //! It's not intended for use outside of `iceberg-rust`. +#[cfg(feature = "tests")] +mod runtime; + #[cfg(feature = "tests")] pub use common::*; +#[cfg(feature = "tests")] +pub use runtime::test_runtime; #[cfg(feature = "tests")] mod common { diff --git a/crates/test_utils/src/runtime.rs b/crates/test_utils/src/runtime.rs new file mode 100644 index 0000000000..28568b67fd --- /dev/null +++ b/crates/test_utils/src/runtime.rs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A shared tokio runtime for tests that build an [`iceberg::table::Table`]. + +use std::sync::OnceLock; + +use iceberg::Runtime; + +/// Returns a process-wide [`Runtime`] suitable for tests that need to construct +/// a [`Table`](iceberg::table::Table) outside a tokio context. +/// +/// The returned [`Runtime`] wraps a single shared multi-thread tokio runtime +/// that is lazily built on first call and lives until process exit. Cloning is +/// cheap, so test code can call this every time it needs a runtime to feed +/// into [`TableBuilder::runtime`](iceberg::table::TableBuilder::runtime). +pub fn test_runtime() -> Runtime { + static TOKIO_RT: OnceLock = OnceLock::new(); + let tokio_rt = TOKIO_RT.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build test tokio runtime") + }); + Runtime::new(tokio_rt) +} From de19a3986fbdf399f09f71cd8c08e2573630c80f Mon Sep 17 00:00:00 2001 From: Dhruv Arya Date: Thu, 1 Oct 2026 08:00:37 +0000 Subject: [PATCH 23/63] fix: avoid lost-wakeup hang in get_equality_delete_predicate_for_delete_file_path (#3268) * fix: avoid lost-wakeup hang in get_equality_delete_predicate_for_delete_file_path The equality-delete loader signals completion with Notify::notify_waiters(), which stores no permit and only wakes Notified futures created before the call. get_equality_delete_predicate_for_delete_file_path observed the Loading state, released the read lock, and only then created the Notified future; a notify_waiters() fired in that window is lost and the await hangs forever. Create the Notified (via notified_owned) while still holding the read lock, so it is guaranteed to precede any notify_waiters() and be delivered. Signed-off-by: Dhruv Arya * docs: explain why building Notified under the read lock prevents the lost wakeup State the mechanism rather than the outcome: notified_owned() snapshots tokio's notify_waiters_calls counter at construction, and holding the read lock orders that snapshot before insert_equality_delete can advance it. This keeps a later refactor from moving notified_owned() out from under the lock unnoticed. Signed-off-by: Dhruv Arya --------- Signed-off-by: Dhruv Arya Co-authored-by: Dhruv Arya --- crates/iceberg/src/arrow/delete_filter.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/arrow/delete_filter.rs b/crates/iceberg/src/arrow/delete_filter.rs index cef81afda8..6f13be71d2 100644 --- a/crates/iceberg/src/arrow/delete_filter.rs +++ b/crates/iceberg/src/arrow/delete_filter.rs @@ -168,17 +168,22 @@ impl DeleteFilter { &self, file_path: &str, ) -> Option { - let notifier = { + // Build the `Notified` while holding the read lock. `notified_owned()` records tokio's + // `notify_waiters_calls` counter at construction and completes on first poll if that + // counter has since advanced. Reading the counter under the lock guarantees it is taken + // before `insert_equality_delete` can advance it via `notify_waiters()`, so the + // notification is never missed even though we `.await` after releasing the lock. + let notified = { match self.state.read().unwrap().equality_deletes.get(file_path) { None => return None, - Some(EqDelState::Loading(notifier)) => notifier.clone(), + Some(EqDelState::Loading(notifier)) => notifier.clone().notified_owned(), Some(EqDelState::Loaded(predicate)) => { return Some(predicate.clone()); } } }; - notifier.notified().await; + notified.await; match self.state.read().unwrap().equality_deletes.get(file_path) { Some(EqDelState::Loaded(predicate)) => Some(predicate.clone()), From 19603861d545f8edbc5016e45cb1bacb0579cbf8 Mon Sep 17 00:00:00 2001 From: kid Date: Thu, 1 Oct 2026 14:54:21 +0000 Subject: [PATCH 24/63] feat: add core COW rewrite primitive (#2752) * feat: add core COW rewrite primitive Add a core copy-on-write rewrite primitive that plans candidate data files, applies a caller-provided RecordBatch rewriter, writes replacement data files, and returns removed/added/unchanged file lists for future overwrite-style commit actions. - Plan candidates through the existing scan path so manifest pruning and delete-file application stay consistent with normal reads; the row predicate is cleared only for the full-file rewrite input. - Stream rewritten batches to the replacement writer: buffer only the prefix before the first changed batch, then open the writer lazily; unchanged files open no writer, fully-deleted files produce no replacement. - Bind the replacement writer and partition key to the planned snapshot's schema so rewrites survive schema evolution. - Build the replacement parquet writer via ParquetWriterBuilder::from_table_properties so replacement files honor the table's write.parquet.* properties, carry the table's encryption manager so encrypted tables are not downgraded to plaintext, and refuse to run when write.format.default is not parquet. - Keep the planner internal; CowRewriteBuilder is the only public entry and CowRewriteFile fields sit behind accessors. Cover delete-style, update-style, no-op, full-file delete, no-match delete, delete-file planning, writer edge cases, schema evolution, multi-source replacement path uniqueness, and partition-value preservation (including the null partition) end-to-end. Part of #2269. Co-Authored-By: Claude * fix: address COW rewrite review comments - derive the effective change flag as `changed || output.is_none()` so a rewriter returning {changed: false, output: None} cannot silently drop rows while leaving the file marked unchanged; add regression tests - treat a candidate whose visible rows are all removed by delete files as removed with no replacement, so it can be compacted away with its delete files instead of landing in unchanged_data_files forever - honor write.object-storage.enabled when building replacement writers (add the missing table property) so object-storage-layout tables keep hash-entropy paths - count output_rows only for rows actually written to replacement files - derive Debug for CowBatchRewrite - document the sync rewriter contract, the read-only CowRewriteFile surface, and the prefix buffer's whole-file worst case - drop avoidable FileScanTask/DataFile clones and replace the writer expect() with a let-else invariant * fix(cow_rewrite): flush kept prefix after a dropped batch A dropped batch could mark a file changed while skipping the Some(output) branch that initialized the writer and flushed kept rows. Open the writer whenever a changed file has buffered or current output, then flush the prefix independently of the current batch output. Cover the keep-first/drop-second data-loss case and full coverage by a real position-delete file; clarify delete-file and schema contracts. * fix(cow_rewrite): guard partition preservation and document rewrite() trade-offs Address the three non-blocking follow-ups from the approval review of apache/iceberg-rust#2752: - Surface the prefix-buffer memory ceiling in the public rewrite() rustdoc: a file that never changes (or whose first change is at EOF) buffers its entire decoded contents, one file at a time, which can be several GB for a compaction-sized file. - Add a debug-only partition guard: replacement files record the source partition verbatim, so a rewriter that mutates a partition-source column would silently write rows under a partition they do not belong to. In debug builds every emitted batch is checked against the source file's partition; release builds are unchanged. - Document in the module docs that keeping the source partition spec and staying unsorted is, like the snapshot-schema choice, a deliberate divergence from Java's RewriteDataFiles / Spark DML. * fix(cow_rewrite): use relocated scan test fixture --------- Co-authored-by: Claude --- crates/iceberg/public-api.txt | 57 + crates/iceberg/src/cow_rewrite/mod.rs | 1667 +++++++++++++++++++ crates/iceberg/src/cow_rewrite/plan.rs | 161 ++ crates/iceberg/src/cow_rewrite/rewriter.rs | 56 + crates/iceberg/src/cow_rewrite/writer.rs | 447 +++++ crates/iceberg/src/lib.rs | 1 + crates/iceberg/src/scan/context.rs | 17 + crates/iceberg/src/scan/mod.rs | 54 +- crates/iceberg/src/scan/task.rs | 10 + crates/iceberg/src/spec/table_properties.rs | 12 + 10 files changed, 2468 insertions(+), 14 deletions(-) create mode 100644 crates/iceberg/src/cow_rewrite/mod.rs create mode 100644 crates/iceberg/src/cow_rewrite/plan.rs create mode 100644 crates/iceberg/src/cow_rewrite/rewriter.rs create mode 100644 crates/iceberg/src/cow_rewrite/writer.rs diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index da0b6b1912..bb4d350e5a 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -175,6 +175,60 @@ impl serde_core::ser::Serialize for iceberg::compression::CompressionCodec pub fn iceberg::compression::CompressionCodec::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> impl<'de> serde_core::de::Deserialize<'de> for iceberg::compression::CompressionCodec pub fn iceberg::compression::CompressionCodec::deserialize>(deserializer: D) -> core::result::Result::Error> +pub mod iceberg::cow_rewrite +pub struct iceberg::cow_rewrite::CowBatchRewrite +pub iceberg::cow_rewrite::CowBatchRewrite::changed: bool +pub iceberg::cow_rewrite::CowBatchRewrite::output: core::option::Option +impl core::fmt::Debug for iceberg::cow_rewrite::CowBatchRewrite +pub fn iceberg::cow_rewrite::CowBatchRewrite::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::cow_rewrite::CowRewriteBuilder<'a> +impl<'a> iceberg::cow_rewrite::CowRewriteBuilder<'a> +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::new(table: &'a iceberg::table::Table) -> Self +pub async fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::rewrite(self) -> iceberg::Result +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_batch_size(self, batch_size: usize) -> Self +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_case_sensitive(self, case_sensitive: bool) -> Self +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_predicate(self, predicate: iceberg::expr::Predicate) -> Self +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_rewriter(self, rewriter: alloc::sync::Arc) -> Self +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_snapshot_id(self, snapshot_id: i64) -> Self +pub struct iceberg::cow_rewrite::CowRewriteFile +impl iceberg::cow_rewrite::CowRewriteFile +pub fn iceberg::cow_rewrite::CowRewriteFile::old_data_file(&self) -> &iceberg::spec::DataFile +pub fn iceberg::cow_rewrite::CowRewriteFile::scan_task(&self) -> &iceberg::scan::FileScanTask +impl core::clone::Clone for iceberg::cow_rewrite::CowRewriteFile +pub fn iceberg::cow_rewrite::CowRewriteFile::clone(&self) -> iceberg::cow_rewrite::CowRewriteFile +impl core::fmt::Debug for iceberg::cow_rewrite::CowRewriteFile +pub fn iceberg::cow_rewrite::CowRewriteFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::cow_rewrite::CowRewriteResult +pub iceberg::cow_rewrite::CowRewriteResult::added_data_files: alloc::vec::Vec +pub iceberg::cow_rewrite::CowRewriteResult::removed_data_files: alloc::vec::Vec +pub iceberg::cow_rewrite::CowRewriteResult::stats: iceberg::cow_rewrite::CowRewriteStats +pub iceberg::cow_rewrite::CowRewriteResult::unchanged_data_files: alloc::vec::Vec +impl iceberg::cow_rewrite::CowRewriteResult +pub fn iceberg::cow_rewrite::CowRewriteResult::has_changes(&self) -> bool +impl core::default::Default for iceberg::cow_rewrite::CowRewriteResult +pub fn iceberg::cow_rewrite::CowRewriteResult::default() -> iceberg::cow_rewrite::CowRewriteResult +impl core::fmt::Debug for iceberg::cow_rewrite::CowRewriteResult +pub fn iceberg::cow_rewrite::CowRewriteResult::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::cow_rewrite::CowRewriteStats +pub iceberg::cow_rewrite::CowRewriteStats::candidate_files: usize +pub iceberg::cow_rewrite::CowRewriteStats::changed_batches: u64 +pub iceberg::cow_rewrite::CowRewriteStats::input_rows: u64 +pub iceberg::cow_rewrite::CowRewriteStats::output_rows: u64 +pub iceberg::cow_rewrite::CowRewriteStats::rewritten_files: usize +pub iceberg::cow_rewrite::CowRewriteStats::unchanged_files: usize +impl core::clone::Clone for iceberg::cow_rewrite::CowRewriteStats +pub fn iceberg::cow_rewrite::CowRewriteStats::clone(&self) -> iceberg::cow_rewrite::CowRewriteStats +impl core::cmp::Eq for iceberg::cow_rewrite::CowRewriteStats +impl core::cmp::PartialEq for iceberg::cow_rewrite::CowRewriteStats +pub fn iceberg::cow_rewrite::CowRewriteStats::eq(&self, other: &iceberg::cow_rewrite::CowRewriteStats) -> bool +impl core::default::Default for iceberg::cow_rewrite::CowRewriteStats +pub fn iceberg::cow_rewrite::CowRewriteStats::default() -> iceberg::cow_rewrite::CowRewriteStats +impl core::fmt::Debug for iceberg::cow_rewrite::CowRewriteStats +pub fn iceberg::cow_rewrite::CowRewriteStats::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::cow_rewrite::CowRewriteStats +impl core::marker::StructuralPartialEq for iceberg::cow_rewrite::CowRewriteStats +pub trait iceberg::cow_rewrite::CowBatchRewriter: core::marker::Send + core::marker::Sync +pub fn iceberg::cow_rewrite::CowBatchRewriter::rewrite_batch(&self, batch: arrow_array::record_batch::RecordBatch) -> iceberg::Result pub mod iceberg::encryption pub mod iceberg::encryption::kms pub struct iceberg::encryption::kms::GeneratedKey @@ -2868,6 +2922,8 @@ pub const iceberg::spec::TableProperties<'_>::PROPERTY_UUID: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_DATA_LOCATION: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_METADATA_PATH: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool @@ -2905,6 +2961,7 @@ pub fn iceberg::spec::TableProperties<'properties>::write_datafusion_fanout_enab pub fn iceberg::spec::TableProperties<'properties>::write_folder_storage_location(&self) -> iceberg::Result> pub fn iceberg::spec::TableProperties<'properties>::write_format_default(&self) -> iceberg::Result pub fn iceberg::spec::TableProperties<'properties>::write_metadata_path(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_enabled(&self) -> iceberg::Result pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_location(&self) -> iceberg::Result> pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_partitioned_paths(&self) -> iceberg::Result pub fn iceberg::spec::TableProperties<'properties>::write_target_file_size_bytes(&self) -> iceberg::Result diff --git a/crates/iceberg/src/cow_rewrite/mod.rs b/crates/iceberg/src/cow_rewrite/mod.rs new file mode 100644 index 0000000000..25d5fb4b8f --- /dev/null +++ b/crates/iceberg/src/cow_rewrite/mod.rs @@ -0,0 +1,1667 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Copy-on-write rewrite primitives. +//! +//! This module plans candidate data files, reads their visible rows, applies a +//! caller-provided batch rewriter, and writes replacement data files. It returns +//! old and new file sets that can be committed by an overwrite-style transaction +//! action. +//! +//! The primitive does not parse SQL and does not commit metadata by itself. +//! Rewriters must emit batches compatible with the schema rows were read in +//! (the planned snapshot's schema) and must preserve each source file's +//! partition values; this primitive does not repartition rewritten rows. +//! +//! Replacement files deliberately mirror their source file's layout instead of +//! re-optimizing it — a deliberate divergence from Java's RewriteDataFiles and +//! Spark DML: they are written in the planned snapshot's schema rather than +//! promoted to the table's current schema, they keep the source file's +//! partition spec rather than being repartitioned into the table's default +//! spec, and they stay unsorted since no sort order is applied. An opt-in +//! promote-to-current-schema rewrite is left for follow-up work. +//! +//! The result carries data files only. A commit adapter consuming these file +//! lists may remove position delete files and deletion vectors only when they +//! exclusively reference removed data files. Equality delete files must be +//! retained while they can still apply to other live data files by sequence +//! number. +//! +//! ```rust,no_run +//! # use std::sync::Arc; +//! # use arrow_array::RecordBatch; +//! # use iceberg::cow_rewrite::{CowBatchRewrite, CowBatchRewriter, CowRewriteBuilder}; +//! # use iceberg::table::Table; +//! # use iceberg::Result; +//! struct KeepAll; +//! +//! impl CowBatchRewriter for KeepAll { +//! fn rewrite_batch(&self, batch: RecordBatch) -> Result { +//! Ok(CowBatchRewrite { +//! output: Some(batch), +//! changed: false, +//! }) +//! } +//! } +//! +//! # async fn example(table: &Table) -> Result<()> { +//! let result = CowRewriteBuilder::new(table) +//! .with_rewriter(Arc::new(KeepAll)) +//! .rewrite() +//! .await?; +//! +//! assert!(!result.has_changes()); +//! # Ok(()) +//! # } +//! ``` + +mod plan; +mod rewriter; +pub(crate) mod writer; + +use std::sync::Arc; + +use arrow_array::RecordBatch; +use futures::TryStreamExt; +pub use plan::CowRewriteFile; +pub use rewriter::{CowBatchRewrite, CowBatchRewriter}; + +use crate::expr::Predicate; +use crate::scan::FileScanTaskStream; +use crate::spec::{DataFile, PartitionKey}; +use crate::table::Table; +use crate::{Error, ErrorKind, Result}; + +/// Counters produced by a copy-on-write rewrite. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CowRewriteStats { + /// Number of candidate files selected by planning. + pub candidate_files: usize, + /// Number of old files that have replacement output or are fully removed. + pub rewritten_files: usize, + /// Number of candidate files that did not change after row rewriting. + pub unchanged_files: usize, + /// Visible input row count read from candidate files. + pub input_rows: u64, + /// Output row count written to replacement files. + /// + /// Rows the rewriter emitted for files that turned out unchanged are not + /// counted, so this always matches the row counts of `added_data_files`. + pub output_rows: u64, + /// Number of input batches that changed, including batches the rewriter + /// dropped entirely (`output: None`) even if it did not flag them. + pub changed_batches: u64, +} + +/// Result of a copy-on-write rewrite operation. +#[derive(Debug, Default)] +pub struct CowRewriteResult { + /// Old data files that should be removed by the commit action. + pub removed_data_files: Vec, + /// New data files that should be added by the commit action. + pub added_data_files: Vec, + /// Candidate files that were read and left unchanged. + /// + /// Files whose visible rows were all removed by delete files are NOT + /// included here: they read as zero rows and are reported in + /// `removed_data_files` with no replacement. A commit adapter may remove + /// position deletes and deletion vectors that exclusively reference these + /// removed files, but must retain equality deletes that can still apply to + /// other live files. + pub unchanged_data_files: Vec, + /// Rewrite counters. + pub stats: CowRewriteStats, +} + +impl CowRewriteResult { + /// Returns true if the rewrite produced any table changes. + pub fn has_changes(&self) -> bool { + !self.removed_data_files.is_empty() || !self.added_data_files.is_empty() + } +} + +/// Builder for orchestrating copy-on-write data file rewrites. +pub struct CowRewriteBuilder<'a> { + table: &'a Table, + predicate: Predicate, + snapshot_id: Option, + batch_size: Option, + case_sensitive: bool, + rewriter: Option>, +} + +impl<'a> CowRewriteBuilder<'a> { + /// Creates a copy-on-write rewrite builder for `table`. + pub fn new(table: &'a Table) -> Self { + Self { + table, + predicate: Predicate::AlwaysTrue, + snapshot_id: None, + batch_size: None, + case_sensitive: true, + rewriter: None, + } + } + + /// Sets the row predicate used to plan candidate files. + pub fn with_predicate(mut self, predicate: Predicate) -> Self { + self.predicate = predicate; + self + } + + /// Sets the snapshot id used to plan candidate files. + pub fn with_snapshot_id(mut self, snapshot_id: i64) -> Self { + self.snapshot_id = Some(snapshot_id); + self + } + + /// Sets the Arrow reader batch size. + pub fn with_batch_size(mut self, batch_size: usize) -> Self { + self.batch_size = Some(batch_size); + self + } + + /// Sets the case sensitivity used to bind the planning predicate. + pub fn with_case_sensitive(mut self, case_sensitive: bool) -> Self { + self.case_sensitive = case_sensitive; + self + } + + /// Sets the record batch rewriter. + pub fn with_rewriter(mut self, rewriter: Arc) -> Self { + self.rewriter = Some(rewriter); + self + } + + /// Plans, reads, rewrites, and writes replacement data files. + /// + /// # Memory usage + /// + /// Batches read before a file's first changed batch are buffered so that + /// no replacement file is emitted for a source file that turns out + /// unchanged. A file that never changes — or whose first change sits at + /// its very end — therefore buffers its entire decoded contents in + /// memory. Files are processed sequentially, so peak usage is one decoded + /// source file at a time, but that can still be several GB for a + /// compaction-sized file. A size-capped fallback that starts writing the + /// replacement once the buffer crosses a threshold is left for follow-up + /// work. + pub async fn rewrite(self) -> Result { + let rewriter = self.rewriter.ok_or_else(|| { + Error::new( + ErrorKind::PreconditionFailed, + "COW rewrite requires a batch rewriter", + ) + })?; + let files = plan::plan_cow_rewrite_files( + self.table, + Some(self.predicate), + self.snapshot_id, + self.case_sensitive, + ) + .await?; + + let mut result = CowRewriteResult { + stats: CowRewriteStats { + candidate_files: files.len(), + ..CowRewriteStats::default() + }, + ..CowRewriteResult::default() + }; + + for file in files { + let CowRewriteFile { + old_data_file, + scan_task, + } = file; + // Schema the rows are read in (the planned snapshot's schema). The + // replacement files must be written with this schema so that batches + // remain compatible when the table's current schema has evolved past + // the snapshot the source files belong to. This preserves the older + // schema in replacement files rather than promoting them to the + // table's current schema during this rewrite. + let write_schema = scan_task.schema_ref(); + let has_delete_files = !scan_task.deletes().is_empty(); + + // Batches produced before the first changed batch. They are buffered + // rather than written immediately because the primitive must not + // emit a replacement file for a source file that turns out to be + // unchanged. Once a changed batch is observed the buffered prefix is + // flushed to the writer and all subsequent batches stream straight + // through. + // + // Worst-case footprint: for a file that never changes (or whose + // first change sits at its very end) the prefix holds the entire + // decoded source file in memory. Files are processed sequentially, + // so peak usage is one file at a time, but that can still be several + // GB for a compaction-sized file. A size-capped fallback that starts + // writing the replacement once the buffer crosses a threshold is + // left for follow-up work. + let mut prefix: Vec = Vec::new(); + let mut file_changed = false; + let mut file_input_rows = 0_u64; + let mut file_output_rows = 0_u64; + let mut writer: Option> = None; + // Debug-only tripwire for the "preserve partition values" + // contract; see `DebugPartitionGuard`. + #[cfg(debug_assertions)] + let partition_guard = + DebugPartitionGuard::new(self.table, &old_data_file, &write_schema)?; + + // Planning already cleared the row predicate (see + // `ManifestEntryContext::into_cow_rewrite_file`), so this task + // reads every row of the source file. + let tasks = Box::pin(futures::stream::iter(vec![Ok(scan_task)])) as FileScanTaskStream; + + // Each candidate file gets its own reader so the per-file prefix + // and lazy-writer semantics stay intact; the delete-file cache is + // therefore also per file, and equality deletes shared by several + // candidates are fetched once per file. + let mut reader_builder = self.table.reader_builder(); + if let Some(batch_size) = self.batch_size { + reader_builder = reader_builder.with_batch_size(batch_size); + } + + let mut batches = reader_builder.build().read(tasks)?.stream(); + while let Some(batch) = batches.try_next().await? { + result.stats.input_rows += batch.num_rows() as u64; + file_input_rows += batch.num_rows() as u64; + + let rewrite = rewriter.rewrite_batch(batch)?; + // `output: None` means the batch is fully removed, which is + // itself a change. Derive the effective flag instead of + // trusting every rewriter to keep `changed` consistent with + // `output` — otherwise a `{changed: false, output: None}` + // batch would silently drop its rows while leaving the file + // marked unchanged. + let changed = rewrite.changed || rewrite.output.is_none(); + if changed { + file_changed = true; + result.stats.changed_batches += 1; + } + + // The guard must see every emitted batch, including ones that + // are only buffered into the prefix before the file flips to + // changed. + #[cfg(debug_assertions)] + if let (Some(guard), Some(output)) = (&partition_guard, &rewrite.output) { + guard.check(output)?; + } + + // A dropped batch can be the first change. Flush any kept + // prefix even when this batch has no output; defer opening the + // writer only if there are no rows to preserve yet. + if file_changed + && writer.is_none() + && (!prefix.is_empty() || rewrite.output.is_some()) + { + let partition_key = + source_partition_key(self.table, &old_data_file, &write_schema)?; + writer = Some( + writer::build_replacement_writer( + self.table, + write_schema.clone(), + Some(partition_key), + ) + .await?, + ); + } + + if let Some(writer) = writer.as_mut() { + for prefix_batch in prefix.drain(..) { + writer.write(prefix_batch).await?; + } + if let Some(output) = rewrite.output { + file_output_rows += output.num_rows() as u64; + writer.write(output).await?; + } + } else if let Some(output) = rewrite.output { + file_output_rows += output.num_rows() as u64; + prefix.push(output); + } + } + + // A candidate whose visible rows were all removed by its delete + // files reads as zero rows and the loop above never runs. Treat it + // the same as a rewriter that dropped every batch — changed with + // no replacement — so the file and its delete files can be + // compacted away instead of being pinned in the table forever. + let fully_removed_by_deletes = + !file_changed && file_input_rows == 0 && has_delete_files; + + if file_changed || fully_removed_by_deletes { + result.stats.rewritten_files += 1; + result.stats.output_rows += file_output_rows; + result.removed_data_files.push(old_data_file); + + if let Some(mut writer) = writer { + let added_data_files = writer.close().await?; + result.added_data_files.extend(added_data_files); + } + // If `writer` is `None`, no visible rows remain after delete + // files and batch rewriting, so no replacement is written. + } else { + result.stats.unchanged_files += 1; + result.unchanged_data_files.push(old_data_file); + // `prefix` is dropped here; no replacement file was written. + } + } + + Ok(result) + } +} + +fn source_partition_key( + table: &Table, + data_file: &DataFile, + schema: &crate::spec::SchemaRef, +) -> Result { + let spec = table + .metadata() + .partition_spec_by_id(data_file.partition_spec_id) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Missing partition spec {} for COW rewrite source file", + data_file.partition_spec_id + ), + ) + })? + .as_ref() + .clone(); + // `PartitionKey::new` does not bind the spec to the schema, so validate the + // binding here: a spec that is incompatible with the planned snapshot + // schema should fail with a clear error instead of producing a bad + // partition path when the writer later calls `PartitionKey::to_path`. + spec.partition_type(schema).map_err(|err| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Cannot bind partition spec {} to the planned snapshot schema for COW rewrite", + data_file.partition_spec_id + ), + ) + .with_source(err) + })?; + + Ok(PartitionKey::new( + spec, + schema.clone(), + data_file.partition().clone(), + )) +} + +/// Debug-only tripwire for the rewriter contract that output batches preserve +/// the source file's partition values: replacement files record the source +/// partition verbatim, so a rewriter that mutates a partition-source column +/// would silently write rows under a partition they do not belong to. Compiled +/// out in release builds, where per-row partition calculation would be too +/// expensive for the rewrite hot path. +#[cfg(debug_assertions)] +struct DebugPartitionGuard { + calculator: crate::arrow::PartitionValueCalculator, + partition_type: crate::spec::StructType, + expected: crate::spec::Struct, +} + +#[cfg(debug_assertions)] +impl DebugPartitionGuard { + /// Builds a guard for one candidate file, or `None` when the source + /// file's partition spec is unpartitioned. Mirrors the spec lookup and + /// schema binding of [`source_partition_key`]. + fn new( + table: &Table, + data_file: &DataFile, + schema: &crate::spec::SchemaRef, + ) -> Result> { + let spec = table + .metadata() + .partition_spec_by_id(data_file.partition_spec_id) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Missing partition spec {} for COW rewrite source file", + data_file.partition_spec_id + ), + ) + })?; + if spec.is_unpartitioned() { + return Ok(None); + } + + Ok(Some(Self { + calculator: crate::arrow::PartitionValueCalculator::try_new(spec, schema)?, + partition_type: spec.partition_type(schema)?, + expected: data_file.partition().clone(), + })) + } + + /// Asserts that every row in `output` computes to the source file's + /// partition under the source spec. + fn check(&self, output: &RecordBatch) -> Result<()> { + let partition_array = self.calculator.calculate(output)?; + let values = crate::arrow::arrow_struct_to_literal(&partition_array, &self.partition_type)?; + for value in values { + debug_assert_eq!( + value, + Some(crate::spec::Literal::Struct(self.expected.clone())), + "COW rewriter must preserve the source file's partition values: \ + replacement files record the source partition verbatim, so \ + mutating a partition-source column writes rows under a \ + partition they do not belong to" + ); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::sync::Arc; + + use arrow_array::{Array, ArrayRef, BooleanArray, Int32Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use futures::TryStreamExt; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use tempfile::TempDir; + + use crate::cow_rewrite::{CowBatchRewrite, CowBatchRewriter, CowRewriteBuilder}; + use crate::io::LocalFsStorageFactory; + use crate::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; + use crate::scan::{FileScanTask, FileScanTaskStream}; + use crate::spec::{ + DataFile, Literal, NestedField, PrimitiveType, Schema, Struct, TableProperties, Transform, + Type, + }; + use crate::table::Table; + use crate::test_utils::scan::TableTestFixture; + use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction}; + use crate::{Catalog, CatalogBuilder, Error, ErrorKind, NamespaceIdent, Result, TableCreation}; + + struct KeepAll; + + impl CowBatchRewriter for KeepAll { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + Ok(CowBatchRewrite { + output: Some(batch), + changed: false, + }) + } + } + + struct DeleteEvenIds; + + impl CowBatchRewriter for DeleteEvenIds { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let ids = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + + let keep = + BooleanArray::from_iter((0..ids.len()).map(|row| Some(ids.value(row) % 2 != 0))); + let filtered = arrow_select::filter::filter_record_batch(&batch, &keep) + .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?; + + Ok(CowBatchRewrite { + changed: filtered.num_rows() != batch.num_rows(), + output: (filtered.num_rows() > 0).then_some(filtered), + }) + } + } + + struct IncrementValueForEvenIds; + + impl CowBatchRewriter for IncrementValueForEvenIds { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let ids = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + let values = batch + .column_by_name("value") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?; + + let mut changed = false; + let updated_values = Int32Array::from_iter((0..values.len()).map(|row| { + let value = values.value(row); + if ids.value(row) % 2 == 0 { + changed = true; + Some(value + 10) + } else { + Some(value) + } + })); + let output = RecordBatch::try_new(batch.schema(), vec![ + batch.column(0).clone(), + Arc::new(updated_values), + ]) + .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?; + + Ok(CowBatchRewrite { + output: Some(output), + changed, + }) + } + } + + struct CowRewriteFixture { + _temp_dir: TempDir, + table: Table, + } + + async fn test_table_with_ids(ids: Vec) -> Result { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_rewrite_fixture".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef + ])?; + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&catalog).await?; + + Ok(CowRewriteFixture { + _temp_dir: temp_dir, + table, + }) + } + + async fn test_table_with_id_batches(batches: Vec>) -> Result { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_rewrite_fixture".to_string()) + .schema(schema) + .properties(HashMap::from([( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "1".to_string(), + )])) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let input = batches.into_iter().map(|ids| { + Ok(RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef, + ])?) + }); + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(input), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&catalog).await?; + + Ok(CowRewriteFixture { + _temp_dir: temp_dir, + table, + }) + } + + async fn test_table_with_id_value_rows(rows: Vec<(i32, i32)>) -> Result { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_rewrite_fixture".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("value", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + let ids = rows.iter().map(|(id, _)| *id).collect::>(); + let values = rows.iter().map(|(_, value)| *value).collect::>(); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef, + Arc::new(Int32Array::from(values)) as ArrayRef, + ])?; + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&catalog).await?; + + Ok(CowRewriteFixture { + _temp_dir: temp_dir, + table, + }) + } + + async fn read_ids(table: &Table, files: &[DataFile]) -> Result> { + let schema = table.metadata().current_schema().clone(); + let project_field_ids = schema + .as_struct() + .fields() + .iter() + .map(|field| field.id) + .collect::>(); + let tasks = files + .iter() + .map(|data_file| { + FileScanTask::builder() + .with_file_size_in_bytes(data_file.file_size_in_bytes()) + .with_start(0) + .with_length(data_file.file_size_in_bytes()) + .with_record_count(Some(data_file.record_count())) + .with_data_file_path(data_file.file_path().to_string()) + .with_data_file_format(data_file.file_format()) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids.clone()) + .with_case_sensitive(true) + .build() + }) + .collect::>(); + let task_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream; + let batches = table + .reader_builder() + .build() + .read(task_stream)? + .stream() + .try_collect::>() + .await?; + + let mut ids = Vec::new(); + for batch in batches { + let column = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + ids.extend((0..column.len()).map(|row| column.value(row))); + } + ids.sort_unstable(); + Ok(ids) + } + + async fn read_ids_and_values(table: &Table, files: &[DataFile]) -> Result> { + let schema = table.metadata().current_schema().clone(); + let project_field_ids = schema + .as_struct() + .fields() + .iter() + .map(|field| field.id) + .collect::>(); + let tasks = files + .iter() + .map(|data_file| { + FileScanTask::builder() + .with_file_size_in_bytes(data_file.file_size_in_bytes()) + .with_start(0) + .with_length(data_file.file_size_in_bytes()) + .with_record_count(Some(data_file.record_count())) + .with_data_file_path(data_file.file_path().to_string()) + .with_data_file_format(data_file.file_format()) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids.clone()) + .with_case_sensitive(true) + .build() + }) + .collect::>(); + let task_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream; + let batches = table + .reader_builder() + .build() + .read(task_stream)? + .stream() + .try_collect::>() + .await?; + + let mut rows = Vec::new(); + for batch in batches { + let ids = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + let values = batch + .column_by_name("value") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?; + rows.extend((0..ids.len()).map(|row| (ids.value(row), values.value(row)))); + } + rows.sort_unstable_by_key(|(id, _)| *id); + Ok(rows) + } + + #[tokio::test] + async fn cow_rewrite_keep_all_produces_no_changes() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 2, 3]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(KeepAll)) + .rewrite() + .await?; + + assert!(!result.has_changes()); + assert_eq!(result.removed_data_files.len(), 0); + assert_eq!(result.added_data_files.len(), 0); + assert_eq!(result.unchanged_data_files.len(), 1); + assert_eq!(result.stats.candidate_files, 1); + assert_eq!(result.stats.unchanged_files, 1); + assert_eq!(result.stats.input_rows, 3); + // Nothing was written, so rows emitted for the unchanged file are not + // counted as output. + assert_eq!(result.stats.output_rows, 0); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_requires_rewriter() -> Result<()> { + let fixture = test_table_with_ids(vec![1]).await?; + + let err = CowRewriteBuilder::new(&fixture.table) + .rewrite() + .await + .expect_err("missing rewriter should fail"); + + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_delete_rows_removes_old_file_and_adds_replacement() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 2, 3, 4]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 4); + assert_eq!(result.stats.output_rows, 2); + + let ids = read_ids(&fixture.table, &result.added_data_files).await?; + assert_eq!(ids, vec![1, 3]); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_update_rows_rewrites_file_with_updated_values() -> Result<()> { + let fixture = + test_table_with_id_value_rows(vec![(1, 10), (2, 20), (3, 30), (4, 40)]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(IncrementValueForEvenIds)) + .rewrite() + .await?; + + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 4); + assert_eq!(result.stats.output_rows, 4); + + let rows = read_ids_and_values(&fixture.table, &result.added_data_files).await?; + assert_eq!(rows, vec![(1, 10), (2, 30), (3, 30), (4, 50)]); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_full_file_delete_removes_old_file_without_replacement() -> Result<()> { + let fixture = test_table_with_ids(vec![2, 4]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 0); + assert_eq!(result.stats.input_rows, 2); + assert_eq!(result.stats.output_rows, 0); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_removes_file_fully_covered_by_position_deletes() -> Result<()> { + let mut fixture = TableTestFixture::new(); + let positions = (0..300).collect::>(); + fixture.setup_multi_row_group_manifest(&positions).await; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(KeepAll)) + .rewrite() + .await?; + + assert_eq!(result.stats.candidate_files, 1); + assert_eq!(result.stats.rewritten_files, 1); + assert_eq!(result.stats.input_rows, 0); + assert_eq!(result.stats.output_rows, 0); + assert_eq!(result.removed_data_files.len(), 1); + assert!(result.added_data_files.is_empty()); + assert!(result.unchanged_data_files.is_empty()); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_delete_no_matching_rows_keeps_old_file() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 3]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert!(!result.has_changes()); + assert_eq!(result.removed_data_files.len(), 0); + assert_eq!(result.added_data_files.len(), 0); + assert_eq!(result.unchanged_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 2); + assert_eq!(result.stats.output_rows, 0); + + Ok(()) + } + + /// Drops every batch while reporting `changed: false`, violating the + /// documented contract. The orchestrator must derive the change from + /// `output: None` itself, otherwise the file would be kept as "unchanged" + /// while its rows were meant to be dropped. + struct SilentFullDrop; + + impl CowBatchRewriter for SilentFullDrop { + fn rewrite_batch(&self, _batch: RecordBatch) -> Result { + Ok(CowBatchRewrite { + output: None, + changed: false, + }) + } + } + + #[tokio::test] + async fn cow_rewrite_none_output_implies_change() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 2]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(SilentFullDrop)) + .rewrite() + .await?; + + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 0); + assert!(result.unchanged_data_files.is_empty()); + assert_eq!(result.stats.rewritten_files, 1); + assert_eq!(result.stats.input_rows, 2); + assert_eq!(result.stats.output_rows, 0); + assert_eq!(result.stats.changed_batches, 1); + + Ok(()) + } + + /// Returns `output: None` with `changed: false` for the first batch, then + /// flags the second batch as changed. The replacement must contain exactly + /// the second batch's rows — the silently dropped first batch must not + /// resurrect, and the original file must be removed. + struct DropFirstBatchSilently { + batches_seen: std::sync::atomic::AtomicUsize, + } + + impl CowBatchRewriter for DropFirstBatchSilently { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let seen = self + .batches_seen + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if seen == 0 { + Ok(CowBatchRewrite { + output: None, + changed: false, + }) + } else { + Ok(CowBatchRewrite { + output: Some(batch), + changed: true, + }) + } + } + } + + #[tokio::test] + async fn cow_rewrite_silent_drop_before_change_loses_no_rows() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 2, 3, 4]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_batch_size(2) + .with_rewriter(Arc::new(DropFirstBatchSilently { + batches_seen: std::sync::atomic::AtomicUsize::new(0), + })) + .rewrite() + .await?; + + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 4); + assert_eq!(result.stats.output_rows, 2); + + let ids = read_ids(&fixture.table, &result.added_data_files).await?; + assert_eq!(ids, vec![3, 4]); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_kept_prefix_survives_later_full_batch_delete() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 3, 2, 4]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_batch_size(2) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 4); + assert_eq!(result.stats.output_rows, 2); + assert_eq!(result.added_data_files.len(), 1); + assert!(result.unchanged_data_files.is_empty()); + assert_eq!( + read_ids(&fixture.table, &result.added_data_files).await?, + vec![1, 3] + ); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_uses_unique_replacement_paths_for_multiple_source_files() -> Result<()> { + let fixture = test_table_with_id_batches(vec![vec![1, 2], vec![3, 4]]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert_eq!(result.stats.candidate_files, 2); + assert_eq!(result.removed_data_files.len(), 2); + assert_eq!(result.added_data_files.len(), 2); + + let added_paths = result + .added_data_files + .iter() + .map(|file| file.file_path().to_string()) + .collect::>(); + let removed_paths = result + .removed_data_files + .iter() + .map(|file| file.file_path().to_string()) + .collect::>(); + + assert_eq!(added_paths.len(), result.added_data_files.len()); + assert!(added_paths.is_disjoint(&removed_paths)); + + let ids = read_ids(&fixture.table, &result.added_data_files).await?; + assert_eq!(ids, vec![1, 3]); + + Ok(()) + } + + #[test] + fn cow_batch_rewriter_is_object_safe() { + let _rewriter: Arc = Arc::new(KeepAll); + } + + #[test] + fn cow_rewrite_result_reports_no_changes() { + let result = crate::cow_rewrite::CowRewriteResult { + removed_data_files: vec![], + added_data_files: vec![], + unchanged_data_files: vec![], + stats: crate::cow_rewrite::CowRewriteStats { + candidate_files: 0, + rewritten_files: 0, + unchanged_files: 0, + input_rows: 0, + output_rows: 0, + changed_batches: 0, + }, + }; + + assert!(!result.has_changes()); + assert_eq!(result.stats.candidate_files, 0); + } + + /// `DeleteIfModThree` keeps rows whose `id` is not divisible by 3. + struct DeleteIfModThree; + + impl CowBatchRewriter for DeleteIfModThree { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let ids = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + + let keep = + BooleanArray::from_iter((0..ids.len()).map(|row| Some(ids.value(row) % 3 != 0))); + let filtered = arrow_select::filter::filter_record_batch(&batch, &keep) + .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?; + + Ok(CowBatchRewrite { + changed: filtered.num_rows() != batch.num_rows(), + output: (filtered.num_rows() > 0).then_some(filtered), + }) + } + } + + /// After adding an optional column (`value`) to the schema, a COW rewrite + /// of the pre-existing data must not fail. The replacement writer must use + /// the schema the batches were read in (the snapshot schema), not the + /// table's evolved current schema, otherwise the parquet writer rejects + /// the input batch whose column set does not include `value`. + #[tokio::test] + async fn cow_rewrite_after_optional_column_add() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog: Arc = Arc::new( + MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?, + ); + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("evolved".to_string()) + .schema(schema) + .build(), + ) + .await?; + + // Write the original data file with the {id}-only schema. + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 1, 2, 3, 4, + ])) as ArrayRef])?; + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&*catalog).await?; + + // Evolve the schema: add an optional `value` column. + let tx = Transaction::new(&table); + let tx = tx + .update_schema() + .add_column(AddColumn::optional( + "value", + Type::Primitive(PrimitiveType::Int), + )) + .apply(tx)?; + let table = tx.commit(&*catalog).await?; + + // The current schema now has {id, value} but the current snapshot's + // schema is still the original {id} schema. + let current_snapshot = table.metadata().current_snapshot().unwrap(); + assert_ne!( + table.metadata().current_schema_id(), + current_snapshot.schema_id().unwrap() + ); + + // Before the fix this fails when the parquet writer rejects the {id}-only + // batch while configured with the {id, value} current schema. + let result = CowRewriteBuilder::new(&table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteIfModThree)) + .rewrite() + .await?; + + // id 3 is divisible by 3 and is dropped; remaining ids are 1, 2, 4. + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 1); + + let ids = read_ids(&table, &result.added_data_files).await?; + assert_eq!(ids, vec![1, 2, 4]); + + Ok(()) + } + + /// Builds a two-file table whose files have disjoint id ranges so predicate + /// metrics pruning can exclude one file entirely. + async fn two_file_table_disjoint_ids() -> Result { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("disjoint".to_string()) + .schema(schema) + .properties(HashMap::from([( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "1".to_string(), + )])) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + // First batch ids 1..=4, second batch ids 100..=103. Force a split via + // tiny target file size so each batch lands in its own data file. + let input = vec![vec![1, 2, 3, 4], vec![100, 101, 102, 103]] + .into_iter() + .map(|ids| { + Ok(RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef, + ])?) + }); + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(input), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&catalog).await?; + + Ok(CowRewriteFixture { + _temp_dir: temp_dir, + table, + }) + } + + /// Predicate-based planning must exclude files whose metrics cannot match + /// the predicate. With `id > 50`, the file holding ids 1..=4 must never be + /// planned as a candidate. + #[tokio::test] + async fn cow_rewrite_predicate_prunes_non_matching_files() -> Result<()> { + let fixture = two_file_table_disjoint_ids().await?; + + let predicate = crate::expr::Reference::new("id").greater_than(crate::spec::Datum::int(50)); + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(predicate) + .with_rewriter(Arc::new(KeepAll)) + .rewrite() + .await?; + + // Only the high-id file should be a candidate. + assert_eq!(result.stats.candidate_files, 1); + // KeepAll does not change anything, so the single candidate is unchanged. + assert_eq!(result.removed_data_files.len(), 0); + assert_eq!(result.added_data_files.len(), 0); + assert_eq!(result.unchanged_data_files.len(), 1); + + // The surviving candidate must be the high-id file (ids 100..=103). + let ids = read_ids(&fixture.table, &result.unchanged_data_files).await?; + assert_eq!(ids, vec![100, 101, 102, 103]); + + Ok(()) + } + + /// Rewrites every batch without altering rows, so every candidate file is + /// replaced. Useful for exercising the replacement-writing path itself. + struct TouchEveryBatch; + + impl CowBatchRewriter for TouchEveryBatch { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + Ok(CowBatchRewrite { + output: Some(batch), + changed: true, + }) + } + } + + /// An end-to-end rewrite on an identity-partitioned table must preserve + /// each source file's partition: the replacement file carries the same + /// partition values and partition spec id, and lands under the same + /// partition directory — including the null partition. + #[tokio::test] + async fn cow_rewrite_preserves_partitions_end_to_end() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "value", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone())) + .add_partition_field("value", "value", Transform::Identity)? + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("partitioned".to_string()) + .schema(schema) + .partition_spec(partition_spec.into_unbound()) + .build(), + ) + .await?; + + // Two source files: ids 1, 2 in the `value=1` partition and ids 3, 4 + // in the null partition. + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("value", DataType::Int32, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + let current_schema = table.metadata().current_schema().clone(); + let default_spec = table.metadata().default_partition_spec().as_ref().clone(); + + let batch_one = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 1])), + ])?; + let one_partition = crate::spec::PartitionKey::new( + default_spec.clone(), + current_schema.clone(), + Struct::from_iter([Some(Literal::int(1))]), + ); + let files_one = super::writer::write_replacement_batches( + &table, + current_schema.clone(), + Some(one_partition), + futures::stream::iter(vec![Ok(batch_one)]), + ) + .await?; + + let batch_null = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![3, 4])) as ArrayRef, + Arc::new(Int32Array::from(vec![None::, None])), + ])?; + let null_partition = crate::spec::PartitionKey::new( + default_spec.clone(), + current_schema.clone(), + Struct::from_iter([None::]), + ); + let files_null = super::writer::write_replacement_batches( + &table, + current_schema.clone(), + Some(null_partition), + futures::stream::iter(vec![Ok(batch_null)]), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .add_data_files([files_one, files_null].concat()) + .apply(tx)?; + let table = tx.commit(&catalog).await?; + + let result = CowRewriteBuilder::new(&table) + .with_rewriter(Arc::new(TouchEveryBatch)) + .rewrite() + .await?; + + assert_eq!(result.stats.candidate_files, 2); + assert_eq!(result.stats.rewritten_files, 2); + assert_eq!(result.removed_data_files.len(), 2); + assert_eq!(result.added_data_files.len(), 2); + assert!(result.unchanged_data_files.is_empty()); + + // Partition values survive the rewrite unchanged. + let one_struct = Struct::from_iter([Some(Literal::int(1))]); + let null_struct = Struct::from_iter([None::]); + let mut removed_partitions = result + .removed_data_files + .iter() + .map(|file| file.partition().clone()) + .collect::>(); + let mut added_partitions = result + .added_data_files + .iter() + .map(|file| file.partition().clone()) + .collect::>(); + removed_partitions.sort_by_key(|partition| partition.is_null_at_index(0)); + added_partitions.sort_by_key(|partition| partition.is_null_at_index(0)); + assert_eq!(removed_partitions, added_partitions); + assert_eq!(added_partitions[0], one_struct); + assert_eq!(added_partitions[1], null_struct); + + for file in &result.added_data_files { + assert_eq!(file.partition_spec_id, 0); + if file.partition() == &one_struct { + assert!(file.file_path().contains("value=1"), "{}", file.file_path()); + } else { + assert!( + file.file_path().contains("value=null"), + "{}", + file.file_path() + ); + } + } + + // Replacement content matches the source rows. + let ids = read_ids(&table, &result.added_data_files).await?; + assert_eq!(ids, vec![1, 2, 3, 4]); + + Ok(()) + } + + /// Mutates the identity-partitioned `value` column while flagging the + /// batch as changed — a violation of the partition-preservation contract. + /// In debug builds the partition guard must trip instead of letting the + /// replacement file record a partition that disagrees with its rows. + #[cfg(debug_assertions)] + struct MutatePartitionColumn; + + #[cfg(debug_assertions)] + impl CowBatchRewriter for MutatePartitionColumn { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let values = batch + .column_by_name("value") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?; + let updated = + Int32Array::from_iter((0..values.len()).map(|row| Some(values.value(row) + 100))); + let output = RecordBatch::try_new(batch.schema(), vec![ + batch.column(0).clone(), + Arc::new(updated), + ]) + .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?; + + Ok(CowBatchRewrite { + output: Some(output), + changed: true, + }) + } + } + + #[cfg(debug_assertions)] + #[tokio::test] + #[should_panic(expected = "must preserve the source file's partition values")] + async fn cow_rewrite_partition_column_mutation_trips_debug_guard() { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await + .unwrap(); + let namespace = NamespaceIdent::new("ns".to_string()); + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap(); + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone())) + .add_partition_field("value", "value", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("partition_guard".to_string()) + .schema(schema) + .partition_spec(partition_spec.into_unbound()) + .build(), + ) + .await + .unwrap(); + + // One source file whose rows all sit in the `value=1` partition. + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("value", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 1])), + ]) + .unwrap(); + let partition_key = crate::spec::PartitionKey::new( + table.metadata().default_partition_spec().as_ref().clone(), + table.metadata().current_schema().clone(), + Struct::from_iter([Some(Literal::int(1))]), + ); + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + Some(partition_key), + futures::stream::iter(vec![Ok(batch)]), + ) + .await + .unwrap(); + + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .add_data_files(data_files) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // The rewriter bumps every `value` by 100, so the output batch + // computes to a different partition than the source file records. + // The debug guard must panic here; without it the rewrite would + // silently commit a replacement file under `value=1` holding rows + // with `value=101`. + let _ = CowRewriteBuilder::new(&table) + .with_rewriter(Arc::new(MutatePartitionColumn)) + .rewrite() + .await; + } +} diff --git a/crates/iceberg/src/cow_rewrite/plan.rs b/crates/iceberg/src/cow_rewrite/plan.rs new file mode 100644 index 0000000000..b96aea635a --- /dev/null +++ b/crates/iceberg/src/cow_rewrite/plan.rs @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use futures::TryStreamExt; + +use crate::scan::FileScanTask; +use crate::spec::DataFile; + +/// A data file selected for COW rewrite. +/// +/// This is a read-only output type: the planning entry points that produce +/// candidates are crate-internal and reached through +/// [`crate::cow_rewrite::CowRewriteBuilder`], and the fields are exposed +/// through accessors only. Follow-up commit-adapter work (overwrite and +/// row-delta actions) consumes planned candidates through those accessors; +/// construction from outside the crate is intentionally not exposed yet and +/// can be added when an adapter actually needs it. +#[derive(Debug, Clone)] +pub struct CowRewriteFile { + /// Original data file from the manifest entry. + pub(crate) old_data_file: DataFile, + /// Full-read task for visible rows in the original file. + pub(crate) scan_task: FileScanTask, +} + +impl CowRewriteFile { + /// Original data file from the manifest entry. + pub fn old_data_file(&self) -> &DataFile { + &self.old_data_file + } + + /// Full-read task for visible rows in the original file. + pub fn scan_task(&self) -> &FileScanTask { + &self.scan_task + } +} + +/// Plan data files that may need copy-on-write rewrite. +pub(crate) async fn plan_cow_rewrite_files( + table: &crate::table::Table, + predicate: Option, + snapshot_id: Option, + case_sensitive: bool, +) -> crate::Result> { + let mut scan_builder = table + .scan() + .select_all() + .with_case_sensitive(case_sensitive); + + if let Some(predicate) = predicate { + scan_builder = scan_builder.with_filter(predicate); + } + + if let Some(snapshot_id) = snapshot_id { + scan_builder = scan_builder.snapshot_id(snapshot_id); + } + + scan_builder + .build()? + .plan_cow_rewrite_files() + .await? + .try_collect() + .await +} + +#[cfg(test)] +mod tests { + use futures::TryStreamExt; + + use crate::Result; + use crate::expr::Predicate; + use crate::test_utils::scan::TableTestFixture; + + #[tokio::test] + async fn cow_planner_returns_old_file_and_full_read_task() -> Result<()> { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + let mut files = + super::plan_cow_rewrite_files(&fixture.table, Some(Predicate::AlwaysTrue), None, true) + .await?; + + assert_eq!(files.len(), 2); + + files.sort_by_key(|file| file.old_data_file.file_path().to_string()); + assert_eq!( + files[0].old_data_file.file_path(), + format!("{}/1.parquet", fixture.table_location) + ); + assert_eq!( + files[1].old_data_file.file_path(), + format!("{}/3.parquet", fixture.table_location) + ); + + for file in files { + assert_eq!( + file.old_data_file.file_path(), + file.scan_task.data_file_path() + ); + assert!(file.scan_task.predicate().is_none()); + assert_eq!( + Some(file.old_data_file.record_count()), + file.scan_task.record_count() + ); + assert_eq!(0, file.scan_task.start()); + assert_eq!( + file.old_data_file.file_size_in_bytes(), + file.scan_task.length() + ); + } + + Ok(()) + } + + #[tokio::test] + async fn cow_planner_preserves_delete_files() -> Result<()> { + let mut fixture = TableTestFixture::new(); + fixture.setup_deadlock_manifests().await; + + let scan = fixture + .table + .scan() + .select_all() + .with_concurrency_limit(1) + .build()?; + + let files = tokio::time::timeout(std::time::Duration::from_secs(5), async { + scan.plan_cow_rewrite_files() + .await? + .try_collect::>() + .await + }) + .await + .expect("COW planning should not deadlock")?; + + assert_eq!(files.len(), 10); + for file in files { + assert_eq!(file.scan_task.deletes().len(), 1); + assert_eq!( + file.scan_task.deletes()[0].file_path, + format!("{}/del.parquet", fixture.table_location) + ); + } + + Ok(()) + } +} diff --git a/crates/iceberg/src/cow_rewrite/rewriter.rs b/crates/iceberg/src/cow_rewrite/rewriter.rs new file mode 100644 index 0000000000..4ed5e3402c --- /dev/null +++ b/crates/iceberg/src/cow_rewrite/rewriter.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow_array::RecordBatch; + +use crate::Result; + +/// Result of rewriting a single record batch. +#[derive(Debug)] +pub struct CowBatchRewrite { + /// Rewritten output batch, or `None` when the input batch is fully removed. + /// + /// Output batches must use the same schema as their input batch — the + /// planned snapshot's schema, which may be older than the table's current + /// schema. Rewriters must also preserve each source file's partition + /// values: this primitive writes replacements into the source file's + /// partition and does not repartition rows. + pub output: Option, + /// Whether the rewrite changed the input batch contents. + /// + /// Set this to `true` whenever `output` differs from the input batch, + /// including filtered rows, updated values, reordered rows, or `None`. + pub changed: bool, +} + +/// Rewrites record batches for copy-on-write operations. +/// +/// `rewrite_batch` is synchronous: it runs on the async runtime thread that +/// drives the read/write pipeline, so implementations must not perform +/// blocking work — async I/O such as catalog enrichment or cross-table +/// lookups is not supported in this contract. The method stays sync (rather +/// than returning a boxed future) so the trait remains object safe for +/// `Arc`. +pub trait CowBatchRewriter: Send + Sync { + /// Rewrites a record batch and reports whether it changed. + /// + /// `output: None` means the batch is fully removed, which is itself a + /// change: the orchestrator treats it as changed regardless of the + /// `changed` flag, so a rewriter cannot accidentally keep dropped rows + /// alive by reporting `changed: false` alongside a `None` output. + fn rewrite_batch(&self, batch: RecordBatch) -> Result; +} diff --git a/crates/iceberg/src/cow_rewrite/writer.rs b/crates/iceberg/src/cow_rewrite/writer.rs new file mode 100644 index 0000000000..ce235e889b --- /dev/null +++ b/crates/iceberg/src/cow_rewrite/writer.rs @@ -0,0 +1,447 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::str::FromStr; + +use uuid::Uuid; + +use crate::Result; +#[cfg(test)] +use crate::spec::DataFile; +use crate::spec::{DataFileFormat, PartitionKey, SchemaRef}; +use crate::table::Table; +use crate::writer::IcebergWriterBuilder; +use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; +use crate::writer::file_writer::ParquetWriterBuilder; +use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator, + ObjectStorageLocationGenerator, +}; +use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + +/// Builds a boxed replacement-data-file writer. +/// +/// `write_schema` is the schema the input batches are encoded in. It must match +/// the schema the rows were read in (the planned snapshot's schema), not the +/// table's possibly-evolved current schema; otherwise the parquet writer will +/// reject batches that lack columns added after the source files were written. +/// +/// Building the writer is cheap: no physical file is opened until the first +/// batch is written, so it is safe to construct one optimistically and only +/// write to it once a source file is known to have changed. +/// +/// The location layout follows `write.object-storage.enabled`: tables that opt +/// into the object-storage layout get hash-entropy paths from +/// [`ObjectStorageLocationGenerator`], everything else gets the default layout. +pub(crate) async fn build_replacement_writer( + table: &Table, + write_schema: SchemaRef, + partition_key: Option, +) -> Result> { + let table_props = table.metadata().table_properties(); + // The replacement writer only produces parquet today; refuse to run on a + // table configured for another format instead of silently writing parquet + // against `write.format.default`. + let write_format_default = table_props.write_format_default()?; + if DataFileFormat::from_str(&write_format_default)? != DataFileFormat::Parquet { + return Err(crate::Error::new( + crate::ErrorKind::FeatureUnsupported, + format!("File format {write_format_default} is not supported for COW rewrite yet!"), + )); + } + let mut parquet_builder = + ParquetWriterBuilder::from_table_properties(&table_props, write_schema)?; + // Replacement files must not silently downgrade an encrypted table to + // plaintext: source rows were read through decryption, so re-encrypt the + // rewritten rows when the table carries an encryption manager. + if let Some(encryption_manager) = table.encryption_manager() { + parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone()); + } + + if table_props.write_object_storage_enabled()? { + let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?; + build_data_file_writer(table, parquet_builder, location_generator, partition_key).await + } else { + let location_generator = DefaultLocationGenerator::new(table.metadata())?; + build_data_file_writer(table, parquet_builder, location_generator, partition_key).await + } +} + +async fn build_data_file_writer( + table: &Table, + parquet_builder: ParquetWriterBuilder, + location_generator: L, + partition_key: Option, +) -> Result> { + let file_name_generator = DefaultFileNameGenerator::new( + format!("cow-rewrite-{}", Uuid::now_v7()), + None, + DataFileFormat::Parquet, + ); + let table_props = table.metadata().table_properties(); + let rolling_builder = RollingFileWriterBuilder::new( + parquet_builder, + table_props.write_target_file_size_bytes()?, + table.file_io().clone(), + location_generator, + file_name_generator, + ); + let data_writer_builder = DataFileWriterBuilder::new(rolling_builder); + Ok(Box::new(data_writer_builder.build(partition_key).await?)) +} + +/// Writes replacement record batches as Iceberg data files. +/// +/// This is a convenience wrapper around [`build_replacement_writer`] that +/// consumes a stream end-to-end. Empty input or all-zero-row batches produce no +/// data files. +#[cfg(test)] +pub(crate) async fn write_replacement_batches( + table: &Table, + write_schema: SchemaRef, + partition_key: Option, + mut batches: S, +) -> Result> +where + S: futures::Stream> + Unpin, +{ + use futures::TryStreamExt as _; + + let mut writer = build_replacement_writer(table, write_schema, partition_key).await?; + + let mut wrote_rows = false; + while let Some(batch) = batches.try_next().await? { + if batch.num_rows() == 0 { + continue; + } + wrote_rows = true; + writer.write(batch).await?; + } + + if !wrote_rows { + return Ok(vec![]); + } + + writer.close().await +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{ArrayRef, Int32Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use tempfile::TempDir; + + use super::write_replacement_batches; + use crate::io::LocalFsStorageFactory; + use crate::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; + use crate::spec::{ + DataContentType, NestedField, PartitionKey, PrimitiveType, Schema, Struct, Transform, Type, + }; + use crate::{Catalog, CatalogBuilder, NamespaceIdent, Result, TableCreation}; + + #[tokio::test] + async fn cow_replacement_writer_outputs_data_files() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_writer".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 1, 2, 3, + ])) as ArrayRef])?; + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + assert_eq!(data_files.len(), 1); + assert_eq!(data_files[0].record_count(), 3); + assert_eq!(data_files[0].content_type(), DataContentType::Data); + + Ok(()) + } + + #[tokio::test] + async fn cow_replacement_writer_skips_empty_input() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_empty_writer".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![]), + ) + .await?; + + assert!(data_files.is_empty()); + + Ok(()) + } + + #[tokio::test] + async fn cow_replacement_writer_skips_zero_row_batches() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_zero_row_writer".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = + RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(Vec::::new())) as ArrayRef, + ])?; + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + assert!(data_files.is_empty()); + + Ok(()) + } + + /// With `write.object-storage.enabled` the replacement file must land in + /// the hash-entropy object storage layout instead of the flat default one. + #[tokio::test] + async fn cow_replacement_writer_honors_object_storage_layout() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_object_storage_writer".to_string()) + .schema(schema) + .properties(HashMap::from([( + crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED + .to_string(), + "true".to_string(), + )])) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 1, 2, 3, + ])) as ArrayRef])?; + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + assert_eq!(data_files.len(), 1); + // Object storage layout: `{data_location}/{entropy dirs}/{file name}`, + // where the entropy dirs are binary strings of 4, 4, 4, and 8 chars. + let path = data_files[0].file_path(); + let after_data = path + .split("/data/") + .nth(1) + .unwrap_or_else(|| panic!("expected /data/ in {path}")); + let segments = after_data.split('/').collect::>(); + assert_eq!(segments.len(), 5, "{path}"); + for (segment, len) in segments[..4].iter().zip([4, 4, 4, 8]) { + assert_eq!(segment.len(), len, "{path}"); + assert!(segment.chars().all(|c| c == '0' || c == '1'), "{path}"); + } + assert!(segments[4].starts_with("cow-rewrite-"), "{path}"); + + Ok(()) + } + + #[tokio::test] + async fn cow_replacement_writer_preserves_partition_key() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone())) + .add_partition_field("id", "id", Transform::Identity)? + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_partitioned_writer".to_string()) + .schema(schema) + .partition_spec(partition_spec.clone().into_unbound()) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 1, 1, 1, + ])) as ArrayRef])?; + let partition_key = PartitionKey::new( + table.metadata().default_partition_spec().as_ref().clone(), + table.metadata().current_schema().clone(), + Struct::from_iter([Some(crate::spec::Literal::int(1))]), + ); + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + Some(partition_key), + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + assert_eq!(data_files.len(), 1); + assert_eq!(data_files[0].partition_spec_id, 0); + assert_eq!( + data_files[0].partition(), + &Struct::from_iter([Some(crate::spec::Literal::int(1))]) + ); + + Ok(()) + } +} diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index bccf82e1fb..9733fac32f 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -83,6 +83,7 @@ pub mod table; mod avro; pub mod cache; pub mod compression; +pub mod cow_rewrite; pub mod io; pub mod spec; diff --git a/crates/iceberg/src/scan/context.rs b/crates/iceberg/src/scan/context.rs index 4635325fbf..d17835afa5 100644 --- a/crates/iceberg/src/scan/context.rs +++ b/crates/iceberg/src/scan/context.rs @@ -172,6 +172,23 @@ impl ManifestEntryContext { .with_sort_order(self.sort_order) .build() } + + /// Consume this `ManifestEntryContext`, returning a COW rewrite file candidate. + /// + /// The scan task keeps everything planned for a normal read (projection, + /// delete files, partition metadata, ...) except the row predicate: the + /// candidate was already selected by that predicate during planning, and + /// the rewrite must read the whole file. + pub(crate) async fn into_cow_rewrite_file(self) -> Result { + let old_data_file = self.manifest_entry.data_file().clone(); + let mut scan_task = self.into_file_scan_task().await?; + scan_task.clear_predicate(); + + Ok(crate::cow_rewrite::CowRewriteFile { + old_data_file, + scan_task, + }) + } } /// PlanContext wraps a [`SnapshotRef`] alongside all the other diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 7d81403969..ccdb5c7d5b 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -24,6 +24,7 @@ use context::*; mod task; use std::collections::HashMap; +use std::future::Future; use std::sync::Arc; use arrow_array::RecordBatch; @@ -397,6 +398,25 @@ pub struct TableScan { impl TableScan { /// Returns a stream of [`FileScanTask`]s. pub async fn plan_files(&self) -> Result { + self.plan_data_files(|ctx| ctx.into_file_scan_task()).await + } + + pub(crate) async fn plan_cow_rewrite_files( + &self, + ) -> Result>> { + self.plan_data_files(|ctx| ctx.into_cow_rewrite_file()) + .await + } + + async fn plan_data_files( + &self, + build_result: F, + ) -> Result>> + where + T: Send + 'static, + F: Fn(ManifestEntryContext) -> Fut + Copy + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { let Some(plan_context) = self.plan_context.as_ref() else { return Ok(Box::pin(futures::stream::empty())); }; @@ -410,8 +430,8 @@ impl TableScan { let (manifest_entry_delete_ctx_tx, manifest_entry_delete_ctx_rx) = channel(concurrency_limit_manifest_files); - // used to stream the results back to the caller - let (file_scan_task_tx, file_scan_task_rx) = channel(concurrency_limit_manifest_entries); + // used to stream the planned data file results back to the caller + let (planned_file_tx, planned_file_rx) = channel(concurrency_limit_manifest_entries); let (delete_file_idx, delete_file_tx) = DeleteFileIndex::new(self.runtime.clone()); @@ -427,9 +447,9 @@ impl TableScan { manifest_entry_delete_ctx_tx, )?; - let mut channel_for_manifest_error = file_scan_task_tx.clone(); - let mut channel_for_data_manifest_entry_error = file_scan_task_tx.clone(); - let mut channel_for_delete_manifest_entry_error = file_scan_task_tx.clone(); + let mut channel_for_manifest_error = planned_file_tx.clone(); + let mut channel_for_data_manifest_entry_error = planned_file_tx.clone(); + let mut channel_for_delete_manifest_entry_error = planned_file_tx.clone(); let rt = self.runtime.clone(); @@ -486,7 +506,7 @@ impl TableScan { let rt_inner = rt.clone(); rt.cpu().spawn(async move { let result = manifest_entry_data_ctx_rx - .map(|me_ctx| Ok((me_ctx, file_scan_task_tx.clone()))) + .map(|me_ctx| Ok((me_ctx, planned_file_tx.clone()))) .try_for_each_concurrent( concurrency_limit_manifest_entries, |(manifest_entry_context, tx)| { @@ -498,6 +518,7 @@ impl TableScan { Self::process_data_manifest_entry( manifest_entry_context, tx, + build_result, ) .await }) @@ -513,7 +534,7 @@ impl TableScan { }); } - Ok(file_scan_task_rx.boxed()) + Ok(planned_file_rx.boxed()) } /// Returns an [`ArrowRecordBatchStream`]. @@ -545,10 +566,15 @@ impl TableScan { self.plan_context.as_ref().map(|x| &x.snapshot) } - async fn process_data_manifest_entry( + async fn process_data_manifest_entry( manifest_entry_context: ManifestEntryContext, - mut file_scan_task_tx: Sender>, - ) -> Result<()> { + mut planned_file_tx: Sender>, + build_result: F, + ) -> Result<()> + where + F: Fn(ManifestEntryContext) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { // skip processing this manifest entry if it has been marked as deleted if !manifest_entry_context.manifest_entry.is_alive() { return Ok(()); @@ -593,10 +619,10 @@ impl TableScan { } // congratulations! the manifest entry has made its way through the - // entire plan without getting filtered out. Create a corresponding - // FileScanTask and push it to the result stream - file_scan_task_tx - .send(Ok(manifest_entry_context.into_file_scan_task().await?)) + // entire plan without getting filtered out. Build the planned file before + // sending so delete-file lookup preserves the original scan timing. + planned_file_tx + .send(Ok(build_result(manifest_entry_context).await?)) .await?; Ok(()) diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 96a322e3d4..97ca7e5181 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -213,6 +213,16 @@ impl FileScanTask { self.predicate.as_ref() } + /// Clears the row predicate of this file scan task. + /// + /// The COW rewrite path uses this after candidate selection: candidates are + /// chosen with the predicate during planning, but each chosen file must then + /// be read in full so the rewrite sees every surviving row. Delete-file + /// application is unaffected. + pub(crate) fn clear_predicate(&mut self) { + self.predicate = None; + } + /// Returns the delete files that may need to be applied to the data file. pub fn deletes(&self) -> &[FileScanTaskDeleteFile] { &self.deletes diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 09bf71c196..f418d3a1de 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -316,6 +316,14 @@ pub struct TableProperties { getter )] write_object_storage_location: Option, + /// Whether new data files use the object storage location layout, which + /// injects hash entropy into file paths to spread object-store prefixes. + #[property( + key = Self::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + default = Self::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT, + getter + )] + write_object_storage_enabled: bool, /// Whether partition values are included in object storage paths. #[property( key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, @@ -530,6 +538,10 @@ impl TableProperties<'_> { pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str = "write.folder-storage.path"; /// Property key for deprecated object storage path, kept as a fallback for compatibility. pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str = "write.object-storage.path"; + /// Property key for enabling the object storage location layout for new data files. + pub const PROPERTY_WRITE_OBJECT_STORAGE_ENABLED: &'static str = "write.object-storage.enabled"; + /// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED`] + pub const PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT: bool = false; /// Property key for controlling whether partition values are included in object storage paths. pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str = "write.object-storage.partitioned-paths"; From fc682d7a050cdc05e2b90ccf04192034dc8f1a1c Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Fri, 2 Oct 2026 09:57:27 +0000 Subject: [PATCH 25/63] fix(spec): Harden decimal and fixed type string parsing (#3256) * fix(spec): Harden decimal and fixed type string parsing deserialize_decimal and deserialize_fixed used trim_start_matches and trim_end_matches, which strip repeated occurrences and never require the prefix, so "decimal(decimal(5, 2)))))" and "fixed[16" both parsed successfully. Precision was also unbounded, so "decimal(50, 2)" was accepted into Schema and TableMetadata even though the crate's own MAX_DECIMAL_PRECISION is 38, and the failure only surfaced much later in Arrow or Avro conversion. Parse strictly with strip_prefix and strip_suffix, and reject precision outside 1..=38 and scale greater than precision. Signed-off-by: Jeremy Schoemaker * fix(spec): Clarify zero precision errors and verify serde roundtrips * test(spec): cover decimal and fixed parsing edge cases * fix(spec): align decimal serde validation with constructor * fix(spec): preserve type parse errors and canonical serde coverage --------- Signed-off-by: Jeremy Schoemaker --- crates/iceberg/src/spec/datatypes.rs | 230 +++++++++++++++++++++++++-- 1 file changed, 214 insertions(+), 16 deletions(-) diff --git a/crates/iceberg/src/spec/datatypes.rs b/crates/iceberg/src/spec/datatypes.rs index 8c4029f7b3..bfee520236 100644 --- a/crates/iceberg/src/spec/datatypes.rs +++ b/crates/iceberg/src/spec/datatypes.rs @@ -191,10 +191,7 @@ impl Type { /// Creates decimal type. #[inline(always)] pub fn decimal(precision: u32, scale: u32) -> Result { - ensure_data_valid!( - precision > 0 && precision <= MAX_DECIMAL_PRECISION, - "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}", - ); + validate_decimal_precision(precision)?; Ok(Type::Primitive(PrimitiveType::Decimal { precision, scale })) } @@ -347,19 +344,40 @@ impl Serialize for PrimitiveType { } } +fn validate_decimal_precision(precision: u32) -> Result<()> { + // Leave scale unconstrained for Java parity; the spec leaves scale > precision open. + ensure_data_valid!(precision > 0, "Decimal precision must be greater than zero",); + ensure_data_valid!( + precision <= MAX_DECIMAL_PRECISION, + "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}", + ); + Ok(()) +} + fn deserialize_decimal<'de, D>(deserializer: D) -> std::result::Result where D: Deserializer<'de> { let s = String::deserialize(deserializer)?; + let malformed = || D::Error::custom(format!("Invalid decimal type: {s}")); + let (precision, scale) = s - .trim_start_matches(r"decimal(") - .trim_end_matches(')') + .strip_prefix("decimal(") + .and_then(|inner| inner.strip_suffix(')')) + .ok_or_else(malformed)? .split_once(',') - .ok_or_else(|| D::Error::custom("Decimal requires precision and scale: {s}"))?; + .ok_or_else(|| D::Error::custom(format!("Decimal requires precision and scale: {s}")))?; + + let (precision, scale) = (precision.trim(), scale.trim()); + if [precision, scale] + .iter() + .any(|token| token.is_empty() || !token.bytes().all(|byte| byte.is_ascii_digit())) + { + return Err(malformed()); + } + let precision: u32 = precision.parse().map_err(D::Error::custom)?; + let scale: u32 = scale.parse().map_err(D::Error::custom)?; + validate_decimal_precision(precision).map_err(D::Error::custom)?; - Ok(PrimitiveType::Decimal { - precision: precision.trim().parse().map_err(D::Error::custom)?, - scale: scale.trim().parse().map_err(D::Error::custom)?, - }) + Ok(PrimitiveType::Decimal { precision, scale }) } fn serialize_decimal( @@ -375,12 +393,16 @@ where fn deserialize_fixed<'de, D>(deserializer: D) -> std::result::Result where D: Deserializer<'de> { - let fixed = String::deserialize(deserializer)? - .trim_start_matches(r"fixed[") - .trim_end_matches(']') - .to_owned(); + let s = String::deserialize(deserializer)?; + + let length = s + .strip_prefix("fixed[") + .and_then(|inner| inner.strip_suffix(']')) + .map(str::trim) + .filter(|length| !length.is_empty() && length.bytes().all(|byte| byte.is_ascii_digit())) + .ok_or_else(|| D::Error::custom(format!("Invalid fixed type: {s}")))?; - fixed + length .parse() .map(PrimitiveType::Fixed) .map_err(D::Error::custom) @@ -1326,6 +1348,182 @@ mod tests { assert_eq!(16, Type::decimal_required_bytes(38).unwrap()); } + #[test] + fn test_reject_malformed_decimal_and_fixed_type_strings() { + for invalid in [ + r#""decimal(50, 2)""#, + r#""decimal(0, 0)""#, + r#""decimal(decimal(5, 2)))""#, + r#""decimal(decimal(5, 2)""#, + r#""decimal(5, 2""#, + r#""decimal(5, 2)))))""#, + r#""decimal(5, 2, 3)""#, + r#""decimal(-5, 2)""#, + r#""decimal(5, -2)""#, + r#""decimal(-5, -2)""#, + r#""decimal(-2, -5)""#, + r#""decimal((5, 2))""#, + r#""decimal[5, 2]""#, + r#""decimal()""#, + r#""fixed[fixed[16]]]""#, + r#""fixed[16""#, + r#""fixed[16]]]""#, + r#""fixed[[16]]""#, + r#""fixed[[16]""#, + r#""fixed(16)""#, + r#""fixed[]""#, + ] { + assert!( + serde_json::from_str::(invalid).is_err(), + "expected {invalid} to be rejected" + ); + } + } + + #[test] + fn test_reject_leading_plus_in_type_strings() { + for invalid in [ + r#""decimal(+5, 2)""#, + r#""decimal(5, +2)""#, + r#""decimal(+5, +2)""#, + r#""fixed[+16]""#, + ] { + assert!(serde_json::from_str::(invalid).is_err(), "{invalid}"); + assert!( + serde_json::from_str::(invalid).is_err(), + "{invalid}" + ); + } + } + + #[test] + fn test_decimal_constructor_json_roundtrip() { + for (precision, scale) in [(1, 0), (5, 5), (5, 8), (38, 38)] { + let decimal = Type::decimal(precision, scale).unwrap(); + let serialized = serde_json::to_string(&decimal).unwrap(); + let reparsed: Type = serde_json::from_str(&serialized).unwrap(); + assert_eq!(reparsed, decimal); + let primitive: PrimitiveType = serde_json::from_str(&serialized).unwrap(); + assert_eq!(Type::Primitive(primitive), decimal); + } + for precision in [0, 39] { + assert!(Type::decimal(precision, 0).is_err()); + let json = format!(r#""decimal({precision}, 0)""#); + assert!(serde_json::from_str::(&json).is_err()); + assert!(serde_json::from_str::(&json).is_err()); + } + } + + #[test] + fn test_empty_decimal_and_fixed_tokens() { + for invalid in [ + r#""decimal(, 2)""#, + r#""decimal(5,)""#, + r#""decimal( , )""#, + r#""fixed[]""#, + r#""fixed[ ]""#, + ] { + assert!(serde_json::from_str::(invalid).is_err(), "{invalid}"); + assert!( + serde_json::from_str::(invalid).is_err(), + "{invalid}" + ); + } + } + + #[test] + fn test_decimal_and_fixed_overflow_errors() { + for invalid in [ + r#""decimal(9999999999, 2)""#, + r#""decimal(5, 9999999999)""#, + r#""fixed[18446744073709551616]""#, + ] { + assert!(serde_json::from_str::(invalid).is_err(), "{invalid}"); + let error = serde_json::from_str::(invalid).unwrap_err(); + assert!( + error + .to_string() + .contains("number too large to fit in target type"), + "unexpected error for {invalid}: {error}" + ); + } + } + + #[test] + fn test_decimal_zero_precision_error() { + let error = serde_json::from_str::(r#""decimal(0, 0)""#).unwrap_err(); + assert!( + error + .to_string() + .contains("Decimal precision must be greater than zero"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_accept_valid_decimal_and_fixed_type_strings() { + for (json, canonical, expected) in [ + ( + r#""decimal(9, 2)""#, + r#""decimal(9, 2)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }), + ), + ( + r#""decimal(38, 10)""#, + r#""decimal(38, 10)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 38, + scale: 10, + }), + ), + ( + r#""decimal(5,2)""#, + r#""decimal(5, 2)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 5, + scale: 2, + }), + ), + ( + r#""decimal(5, 0)""#, + r#""decimal(5, 0)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 5, + scale: 0, + }), + ), + ( + r#""decimal(5, 5)""#, + r#""decimal(5, 5)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 5, + scale: 5, + }), + ), + ( + r#""fixed[16]""#, + r#""fixed[16]""#, + Type::Primitive(PrimitiveType::Fixed(16)), + ), + ] { + check_type_serde_roundtrip_value(json, canonical, expected); + } + } + + fn check_type_serde_roundtrip_value(json: &str, canonical: &str, expected_type: Type) { + let parsed: Type = serde_json::from_str(json).unwrap(); + assert_eq!(parsed, expected_type); + let serialized = serde_json::to_string(&parsed).unwrap(); + assert_eq!(serialized, canonical); + let primitive: PrimitiveType = serde_json::from_str(json).unwrap(); + assert_eq!(serde_json::to_string(&primitive).unwrap(), canonical); + let reparsed: Type = serde_json::from_str(&serialized).unwrap(); + assert_eq!(reparsed, expected_type); + } + #[test] fn test_primitive_type_compatible() { let pairs = vec![ From e9c22bf7148a2fadd8dc94879371866dea89c883 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Fri, 2 Oct 2026 09:57:45 +0000 Subject: [PATCH 26/63] perf(arrow): skip rebuilding projected arrays without parent nulls (#3282) * perf(arrow): skip rebuilding top-level projected arrays * perf(arrow): drop in-crate benchmark harness * perf(arrow): skip rebuilding nested projected arrays * perf(arrow): tighten projector zero-copy path * perf(arrow): polish projector zero-copy path --- .../src/arrow/record_batch_projector.rs | 160 ++++++++++++++++-- 1 file changed, 144 insertions(+), 16 deletions(-) diff --git a/crates/iceberg/src/arrow/record_batch_projector.rs b/crates/iceberg/src/arrow/record_batch_projector.rs index d0802ee53b..6dda6b0f46 100644 --- a/crates/iceberg/src/arrow/record_batch_projector.rs +++ b/crates/iceberg/src/arrow/record_batch_projector.rs @@ -17,7 +17,7 @@ use std::sync::Arc; -use arrow_array::{ArrayRef, RecordBatch, StructArray, make_array}; +use arrow_array::{Array, ArrayRef, RecordBatch, StructArray, make_array}; use arrow_buffer::NullBuffer; use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; @@ -169,28 +169,33 @@ impl RecordBatchProjector { /// Do projection with columns pub fn project_column(&self, batch: &[ArrayRef]) -> Result> { - self.field_indices - .iter() - .map(|index_vec| Self::get_column_by_field_index(batch, index_vec)) - .collect::>>() + let mut columns = Vec::with_capacity(self.field_indices.len()); + for index_vec in &self.field_indices { + columns.push(Self::get_column_by_field_index(batch, index_vec)?); + } + Ok(columns) } fn get_column_by_field_index(batch: &[ArrayRef], field_index: &[usize]) -> Result { let mut rev_iterator = field_index.iter().rev(); - let mut array = batch[*rev_iterator.next().unwrap()].clone(); - let mut null_buffer = array.logical_nulls(); + let mut array = &batch[*rev_iterator.next().unwrap()]; + let mut ancestor_nulls = None; for idx in rev_iterator { - array = array + let struct_array = array .as_any() .downcast_ref::() - .ok_or(Error::new( - ErrorKind::Unexpected, - "Cannot convert Array to StructArray", - ))? - .column(*idx) - .clone(); - null_buffer = NullBuffer::union(null_buffer.as_ref(), array.logical_nulls().as_ref()); + .ok_or_else(|| { + Error::new(ErrorKind::Unexpected, "Cannot convert Array to StructArray") + })?; + ancestor_nulls = NullBuffer::union(ancestor_nulls.as_ref(), struct_array.nulls()); + array = struct_array.column(*idx); } + let Some(ancestor_nulls) = ancestor_nulls else { + // Iceberg leaf types handled here have logical_nulls() == nulls(), so no null + // materialization is needed when no ancestor contributes nulls. + return Ok(Arc::clone(array)); + }; + let null_buffer = NullBuffer::union(Some(&ancestor_nulls), array.logical_nulls().as_ref()); Ok(make_array( array.to_data().into_builder().nulls(null_buffer).build()?, )) @@ -201,13 +206,40 @@ impl RecordBatchProjector { mod test { use std::sync::Arc; - use arrow_array::{ArrayRef, Int32Array, RecordBatch, StringArray, StructArray}; + use arrow_array::{Array, ArrayRef, Int32Array, RecordBatch, StringArray, StructArray}; + use arrow_buffer::NullBuffer; use arrow_schema::{DataType, Field, Fields, Schema}; use crate::arrow::record_batch_projector::RecordBatchProjector; use crate::spec::{NestedField, PrimitiveType, Schema as IcebergSchema, Type}; use crate::{Error, ErrorKind}; + fn nested_projector() -> (RecordBatchProjector, Field, Field) { + let leaf_field = Field::new("leaf", DataType::Int32, true); + let inner_field = Field::new( + "inner", + DataType::Struct(Fields::from(vec![leaf_field.clone()])), + true, + ); + let outer_field = Field::new( + "outer", + DataType::Struct(Fields::from(vec![inner_field.clone()])), + true, + ); + let schema = Arc::new(Schema::new(vec![outer_field])); + let field_id_fetch_func = |field: &Field| -> crate::Result> { + match field.name().as_str() { + "outer" => Ok(Some(1)), + "inner" => Ok(Some(2)), + "leaf" => Ok(Some(3)), + _ => Ok(None), + } + }; + let projector = + RecordBatchProjector::new(schema, &[3], field_id_fetch_func, |_| true).unwrap(); + (projector, inner_field, leaf_field) + } + #[test] fn test_record_batch_projector_nested_level() { let inner_fields = vec![ @@ -271,6 +303,102 @@ mod test { assert_eq!(projected_inner_int_array.values(), &[4, 5, 6]); } + #[test] + fn test_record_batch_projector_top_level_nullable_column() { + let iceberg_schema = IcebergSchema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let projector = + RecordBatchProjector::from_iceberg_schema(Arc::new(iceberg_schema), &[1]).unwrap(); + let input = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef; + + let projected = projector + .project_column(std::slice::from_ref(&input)) + .unwrap(); + assert!(Arc::ptr_eq(&projected[0], &input)); + let projected_array = projected[0].as_any().downcast_ref::().unwrap(); + + assert_eq!(projected_array.value(0), 10); + assert_eq!(projected_array.null_count(), 1); + assert!(projected_array.is_null(1)); + assert_eq!(projected_array.value(2), 30); + } + + #[test] + fn test_record_batch_projector_nested_nullable_leaf_without_parent_nulls() { + let (projector, inner_field, leaf_field) = nested_projector(); + let leaf = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef; + let inner = Arc::new(StructArray::new( + Fields::from(vec![leaf_field]), + vec![leaf.clone()], + None, + )) as ArrayRef; + let outer = Arc::new(StructArray::new( + Fields::from(vec![inner_field]), + vec![inner], + None, + )) as ArrayRef; + + let projected = projector.project_column(&[outer]).unwrap(); + assert!(Arc::ptr_eq(&projected[0], &leaf)); + } + + #[test] + fn test_record_batch_projector_nested_all_valid_slice_reuses_leaf() { + let (projector, inner_field, leaf_field) = nested_projector(); + let leaf = Arc::new(Int32Array::from(vec![Some(0), Some(10), None, Some(30)])) as ArrayRef; + let inner = Arc::new(StructArray::new( + Fields::from(vec![leaf_field]), + vec![leaf], + None, + )) as ArrayRef; + let outer = StructArray::new( + Fields::from(vec![inner_field]), + vec![inner], + Some(NullBuffer::from(vec![false, true, true, true])), + ) + .slice(1, 3); + assert_eq!(outer.null_count(), 0); + assert!(outer.nulls().is_some()); + + let sliced_inner = outer + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let sliced_leaf = sliced_inner.column(0).clone(); + let projected = projector + .project_column(&[Arc::new(outer) as ArrayRef]) + .unwrap(); + assert!(Arc::ptr_eq(&projected[0], &sliced_leaf)); + } + + #[test] + fn test_record_batch_projector_propagates_nested_parent_nulls() { + let (projector, inner_field, leaf_field) = nested_projector(); + let leaf = Arc::new(Int32Array::from(vec![Some(10), Some(20), Some(30), None])) as ArrayRef; + let inner = Arc::new(StructArray::new( + Fields::from(vec![leaf_field]), + vec![leaf], + Some(NullBuffer::from(vec![true, true, false, true])), + )) as ArrayRef; + let outer = Arc::new(StructArray::new( + Fields::from(vec![inner_field]), + vec![inner], + Some(NullBuffer::from(vec![true, false, true, true])), + )) as ArrayRef; + + let projected = projector.project_column(&[outer]).unwrap(); + assert_eq!( + projected[0].as_ref(), + &Int32Array::from(vec![Some(10), None, None, None]) + ); + } + #[test] fn test_field_not_found() { let inner_fields = vec![ From ffa864bf2fd3ba27ff221746f993d4bf7810440b Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 2 Oct 2026 13:57:07 +0000 Subject: [PATCH 27/63] fix(reader): page-index pruning drops rows for INT32/INT64 decimals (#3314) Page-index bounds for INT32/INT64 columns were built as Int/Long literals regardless of the Iceberg field type. For decimal fields, query datums use Int128, and Datum::partial_cmp only compares decimals when both sides are Int128, so every comparison returned None. This silently pruned all matching pages for inequality and IN predicates when page-index row selection was enabled. Convert INT32/INT64 page bounds to Int128 for decimal fields, matching the row-group statistics path. Follow-up: FIXED_LEN_BYTE_ARRAY decimals (precision > 18) still get no page-index pruning, even though the row-group stats path decodes them. This only over-selects pages (never drops rows) and is pre-existing. Decoding them here later must guard against truncated ColumnIndex bounds (column_index_truncate_length). Closes #3264 Co-authored-by: Fokko Driesprong --- .../src/expr/visitors/page_index_evaluator.rs | 265 +++++++++++++++++- 1 file changed, 259 insertions(+), 6 deletions(-) diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index 37d9c6a233..d494e5426f 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -276,8 +276,8 @@ impl<'a> PageIndexEvaluator<'a> { .zip(row_counts.iter()) .map(|((i, (min, max)), &row_count)| { predicate( - min.map(|&val| Datum::new(field_type.clone(), PrimitiveLiteral::Int(val))), - max.map(|&val| Datum::new(field_type.clone(), PrimitiveLiteral::Int(val))), + min.map(|&val| Self::int32_bound_to_datum(field_type, val)), + max.map(|&val| Self::int32_bound_to_datum(field_type, val)), PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) @@ -289,8 +289,8 @@ impl<'a> PageIndexEvaluator<'a> { .zip(row_counts.iter()) .map(|((i, (min, max)), &row_count)| { predicate( - min.map(|&val| Datum::new(field_type.clone(), PrimitiveLiteral::Long(val))), - max.map(|&val| Datum::new(field_type.clone(), PrimitiveLiteral::Long(val))), + min.map(|&val| Self::int64_bound_to_datum(field_type, val)), + max.map(|&val| Self::int64_bound_to_datum(field_type, val)), PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) @@ -410,6 +410,30 @@ impl<'a> PageIndexEvaluator<'a> { Ok(Some(result?)) } + /// Converts an `INT32` page bound into a [`Datum`] according to the field's + /// primitive type. + fn int32_bound_to_datum(field_type: &PrimitiveType, val: i32) -> Datum { + match field_type { + PrimitiveType::Decimal { .. } => Datum::new( + field_type.clone(), + PrimitiveLiteral::Int128(i128::from(val)), + ), + _ => Datum::new(field_type.clone(), PrimitiveLiteral::Int(val)), + } + } + + /// Converts an `INT64` page bound into a [`Datum`] according to the field's + /// primitive type. + fn int64_bound_to_datum(field_type: &PrimitiveType, val: i64) -> Datum { + match field_type { + PrimitiveType::Decimal { .. } => Datum::new( + field_type.clone(), + PrimitiveLiteral::Int128(i128::from(val)), + ), + _ => Datum::new(field_type.clone(), PrimitiveLiteral::Long(val)), + } + } + /// Converts a `BYTE_ARRAY` page bound into a [`Datum`] according to the /// field's primitive type. Parquet stores Iceberg `string` and `binary` /// bounds as `BYTE_ARRAY`. @@ -832,7 +856,8 @@ mod tests { use std::sync::Arc; use arrow_array::{ - ArrayRef, FixedSizeBinaryArray, Float32Array, LargeBinaryArray, RecordBatch, StringArray, + ArrayRef, Decimal128Array, FixedSizeBinaryArray, Float32Array, LargeBinaryArray, + RecordBatch, StringArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use parquet::arrow::ArrowWriter; @@ -846,6 +871,7 @@ mod tests { use super::PageIndexEvaluator; use crate::expr::{Bind, Reference}; + use crate::spec::decimal_utils::decimal_from_i128_with_scale; use crate::spec::{Datum, NestedField, PrimitiveType, Schema, Type}; use crate::{ErrorKind, Result}; @@ -1036,6 +1062,55 @@ mod tests { Ok((metadata, temp_file)) } + /// Creates a single-column `Decimal128(precision, scale)` parquet file. + /// Parquet encodes precision <= 9 as `INT32` and precision 10..=18 as + /// `INT64`, so `precision` selects which page-index encoding is exercised. + /// Writes one 1024-row page per value in `unscaled`, so page bounds + /// partition the value range. + fn create_decimal_parquet_file( + precision: u8, + scale: i8, + unscaled: &[i128], + ) -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "col_decimal", + DataType::Decimal128(precision, scale), + true, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + for &unscaled in unscaled { + let array = Arc::new( + Decimal128Array::from_iter_values(std::iter::repeat_n(unscaled, 1024)) + .with_precision_and_scale(precision, scale) + .unwrap(), + ) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + // Write rows one at a time so the writer splits into per-value pages. + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + /// Get the test metadata components for testing fn get_test_metadata( metadata: &ParquetMetaData, @@ -1392,7 +1467,7 @@ mod tests { // A predicate that would prune every page if the bounds were decoded. let filter = Reference::new("col_decimal") .greater_than(Datum::decimal_with_precision( - crate::spec::decimal_utils::decimal_from_i128_with_scale(99999, 2), + decimal_from_i128_with_scale(99999, 2), 10, )?) .bind(iceberg_schema.clone(), false)?; @@ -1651,6 +1726,184 @@ mod tests { Ok(()) } + #[test] + fn eval_inequality_prunes_int32_decimal_pages() -> Result<()> { + // precision 9 -> Parquet INT32 page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(9, 2, &[100, 200, 300, 400])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(9, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. `> 2.50` keeps the pages whose + // upper bound exceeds 2.50 (3.00 and 4.00). + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(250, 2, 9)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(2048) + ]); + + Ok(()) + } + + #[test] + fn eval_in_prunes_int32_decimal_pages() -> Result<()> { + // precision 9 -> Parquet INT32 page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(9, 2, &[100, 200, 300, 400])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(9, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. IN (2.00, 4.00) keeps only the + // pages whose single value is one of the literals. + let filter = Reference::new("col_decimal") + .is_in([decimal_datum(200, 2, 9)?, decimal_datum(400, 2, 9)?]) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(1024), + RowSelector::skip(1024), + RowSelector::select(1024), + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_int64_decimal_pages() -> Result<()> { + // precision 18 -> Parquet INT64 page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(18, 2, &[100, 200, 300, 400])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(18, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. `> 2.50` keeps the pages whose + // upper bound exceeds 2.50 (3.00 and 4.00). + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(250, 2, 18)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(2048) + ]); + + Ok(()) + } + + #[test] + fn eval_in_prunes_int64_decimal_pages() -> Result<()> { + // precision 18 -> Parquet INT64 page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(18, 2, &[100, 200, 300, 400])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(18, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. IN (2.00, 4.00) keeps only the + // pages whose single value is one of the literals. + let filter = Reference::new("col_decimal") + .is_in([decimal_datum(200, 2, 18)?, decimal_datum(400, 2, 18)?]) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(1024), + RowSelector::skip(1024), + RowSelector::select(1024), + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_negative_decimal_pages_at_boundary() -> Result<()> { + // precision 9 -> Parquet INT32 page bounds, spanning negative values. + let (metadata, _temp_file) = create_decimal_parquet_file(9, 2, &[-400, -200, 100, 300])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(9, 2)?; + + // Pages hold -4.00, -2.00, 1.00, 3.00. `>= -2.00` keeps the pages whose + // upper bound is at least -2.00, including page 1 whose bound equals it. + let filter = Reference::new("col_decimal") + .greater_than_or_equal_to(decimal_datum(-200, 2, 9)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(3072) + ]); + + Ok(()) + } + + /// Builds a decimal query [`Datum`] from an `unscaled` mantissa, matching + /// the field's `scale` and `precision`. + fn decimal_datum(unscaled: i128, scale: u32, precision: u32) -> Result { + Datum::decimal_with_precision(decimal_from_i128_with_scale(unscaled, scale), precision) + } + + fn build_decimal_schema_and_field_map( + precision: u32, + scale: u32, + ) -> Result<(Arc, HashMap)> { + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_decimal", + Type::Primitive(PrimitiveType::Decimal { precision, scale }), + true, + ))]) + .build()?, + ); + + Ok((iceberg_schema, HashMap::from_iter([(1, 0)]))) + } + fn build_iceberg_schema_and_field_map() -> Result<(Arc, HashMap)> { let iceberg_schema = Schema::builder() .with_fields([ From 3c91b3942ab40ec005dfbb9fdbccea848392cccb Mon Sep 17 00:00:00 2001 From: NoahKusaba <97474796+NoahKusaba@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:56:13 +0000 Subject: [PATCH 28/63] feat(catalog-loader): add with_runtime to BoxedCatalogBuilder (#3286) * init * address review: doc sibling methods, tidy test, note break in changelog --- CHANGELOG.md | 1 + crates/catalog/loader/public-api.txt | 2 + crates/catalog/loader/src/lib.rs | 94 ++++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83eeeb31ea..f357a03e57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). * `EncryptedOutputFile::key_metadata()` is replaced by `key_metadata_with_saved_file_metadata(&FileMetadata)`. Pass the metadata returned by `write()` or the writer's `close()` to include the stored length before encoding key metadata. * `EncryptedInputFile::metadata()` is now synchronous and derives the plaintext size from key metadata without a storage stat. Remove `.await` from calls to this method. * AGS1 readers now require `StandardKeyMetadata::file_length` and reject missing or invalid lengths without falling back to a storage stat. AGS1-encrypted manifests, manifest lists, and Puffin files written by earlier development builds without this field must be rewritten using a build that can still read them before upgrading. This matches the Java client's read contract. +* `iceberg_catalog_loader::BoxedCatalogBuilder` has a new required method, `with_runtime`. Types implementing `CatalogBuilder` get it through the blanket impl; types implementing `BoxedCatalogBuilder` directly (for example, a wrapper around another `Box`) must implement it, typically by forwarding the runtime to the builder they wrap. ## [v0.10.1] - 2026-07-28 diff --git a/crates/catalog/loader/public-api.txt b/crates/catalog/loader/public-api.txt index 1efaf831df..4e3c288614 100644 --- a/crates/catalog/loader/public-api.txt +++ b/crates/catalog/loader/public-api.txt @@ -7,10 +7,12 @@ pub fn iceberg_catalog_loader::CatalogLoader<'a>::from(s: &'a str) -> Self pub trait iceberg_catalog_loader::BoxedCatalogBuilder: core::marker::Send pub fn iceberg_catalog_loader::BoxedCatalogBuilder::load<'async_trait>(self: alloc::boxed::Box, name: alloc::string::String, props: std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait pub fn iceberg_catalog_loader::BoxedCatalogBuilder::with_kms_client_factory(self: alloc::boxed::Box, kms_client_factory: alloc::sync::Arc) -> alloc::boxed::Box +pub fn iceberg_catalog_loader::BoxedCatalogBuilder::with_runtime(self: alloc::boxed::Box, runtime: iceberg::runtime::Runtime) -> alloc::boxed::Box pub fn iceberg_catalog_loader::BoxedCatalogBuilder::with_storage_factory(self: alloc::boxed::Box, storage_factory: alloc::sync::Arc) -> alloc::boxed::Box impl iceberg_catalog_loader::BoxedCatalogBuilder for T pub fn T::load<'async_trait>(self: alloc::boxed::Box, name: alloc::string::String, props: std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait pub fn T::with_kms_client_factory(self: alloc::boxed::Box, kms_client_factory: alloc::sync::Arc) -> alloc::boxed::Box +pub fn T::with_runtime(self: alloc::boxed::Box, runtime: iceberg::runtime::Runtime) -> alloc::boxed::Box pub fn T::with_storage_factory(self: alloc::boxed::Box, storage_factory: alloc::sync::Arc) -> alloc::boxed::Box pub fn iceberg_catalog_loader::load(type: &str) -> iceberg::error::Result> pub fn iceberg_catalog_loader::supported_types() -> alloc::vec::Vec<&'static str> diff --git a/crates/catalog/loader/src/lib.rs b/crates/catalog/loader/src/lib.rs index d45decc40e..3993de6764 100644 --- a/crates/catalog/loader/src/lib.rs +++ b/crates/catalog/loader/src/lib.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use async_trait::async_trait; use iceberg::encryption::kms::KmsClientFactory; use iceberg::io::StorageFactory; -use iceberg::{Catalog, CatalogBuilder, Error, ErrorKind, Result}; +use iceberg::{Catalog, CatalogBuilder, Error, ErrorKind, Result, Runtime}; use iceberg_catalog_glue::GlueCatalogBuilder; use iceberg_catalog_hms::HmsCatalogBuilder; use iceberg_catalog_rest::RestCatalogBuilder; @@ -47,16 +47,24 @@ pub fn supported_types() -> Vec<&'static str> { #[async_trait] pub trait BoxedCatalogBuilder: Send { + /// Sets the storage factory used to build the catalog's `FileIO`; see + /// [`CatalogBuilder::with_storage_factory`]. fn with_storage_factory( self: Box, storage_factory: Arc, ) -> Box; + /// Sets the KMS client factory used to enable table encryption; see + /// [`CatalogBuilder::with_kms_client_factory`]. fn with_kms_client_factory( self: Box, kms_client_factory: Arc, ) -> Box; + /// Sets the runtime the catalog, and the tables it creates, spawn their + /// tasks on; see [`CatalogBuilder::with_runtime`]. + fn with_runtime(self: Box, runtime: Runtime) -> Box; + async fn load( self: Box, name: String, @@ -83,6 +91,10 @@ impl BoxedCatalogBuilder for T { )) } + fn with_runtime(self: Box, runtime: Runtime) -> Box { + Box::new(CatalogBuilder::with_runtime(*self, runtime)) + } + async fn load( self: Box, name: String, @@ -138,13 +150,16 @@ impl CatalogLoader<'_> { #[cfg(test)] mod tests { use std::collections::HashMap; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; - use iceberg::io::LocalFsStorageFactory; + use iceberg::encryption::kms::KmsClientFactory; + use iceberg::io::{LocalFsStorageFactory, StorageFactory}; + use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalog, MemoryCatalogBuilder}; + use iceberg::{CatalogBuilder, Result, Runtime}; use sqlx::migrate::MigrateDatabase; use tempfile::TempDir; - use crate::{CatalogLoader, load}; + use crate::{BoxedCatalogBuilder, CatalogLoader, load}; #[tokio::test] async fn test_load_unsupported_catalog() { @@ -287,6 +302,77 @@ mod tests { assert!(catalog.is_ok()); } + /// A memory catalog builder that records the runtime it is given. + #[derive(Debug, Default)] + struct RuntimeRecordingBuilder { + runtime: Arc>>, + } + + impl CatalogBuilder for RuntimeRecordingBuilder { + type C = MemoryCatalog; + + fn with_storage_factory(self, _storage_factory: Arc) -> Self { + self + } + + fn with_kms_client_factory(self, _kms_client_factory: Arc) -> Self { + self + } + + fn with_runtime(self, runtime: Runtime) -> Self { + *self.runtime.lock().unwrap() = Some(runtime); + self + } + + fn load( + self, + name: impl Into, + props: HashMap, + ) -> impl Future> + Send { + MemoryCatalogBuilder::default().load(name, props) + } + } + + #[test] + fn test_with_runtime_reaches_the_catalog_builder() { + let tokio_runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .thread_name("loader-test-runtime") + .enable_all() + .build() + .unwrap(); + let recorded = Arc::new(Mutex::new(None)); + let builder: Box = Box::new(RuntimeRecordingBuilder { + runtime: recorded.clone(), + }); + + tokio_runtime.block_on(async { + builder + .with_runtime(Runtime::new(&tokio_runtime)) + .load( + "memory".to_string(), + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), temp_path())]), + ) + .await + .unwrap(); + + // The builder got the runtime passed to the boxed builder: its + // tasks run on that runtime's threads. + let runtime = recorded.lock().unwrap().take().unwrap(); + let thread = runtime + .io() + .spawn(async { std::thread::current().name().map(str::to_string) }) + .await + .unwrap(); + assert!( + thread + .as_deref() + .is_some_and(|n| n.starts_with("loader-test-runtime")), + "got: {thread:?}" + ); + }); + } + #[tokio::test] async fn test_error_message_includes_supported_types() { let err = match load("does-not-exist") { From 8e81f33daa31e495b94ef9e0bc5ff614355f451c Mon Sep 17 00:00:00 2001 From: Oleks V Date: Sun, 4 Oct 2026 15:36:50 +0000 Subject: [PATCH 29/63] ci: check dependency licenses with cargo-deny on every PR (#3293) * ci: check dependency licenses with cargo-deny on every PR Run `dev/release/dependencies.sh check` in the `lint` job, so a dependency whose license is not allowed by `deny.toml` fails the pull request instead of being found at release time. Set `[graph] all-features = true` in `deny.toml`. Without it, cargo-deny only follows default features. Since #3143 that graph no longer includes the optional OpenDAL backends, so the check skipped the MPL-2.0 crates that `deny.toml` has exceptions for. Closes #3234. * Tighten docs/comments, reuse dependencies.sh in create_rc.sh - Trim comments and docs that over-explain. - Condense the CONTRIBUTING.md license guidance. - create_rc.sh calls dependencies.sh check so the release uses the pinned cargo-deny version, same as CI. - Drop unused bzip2-1.0.6 from the deny.toml allow-list. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Kevin Liu Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 5 ++++- CONTRIBUTING.md | 8 ++++++++ Makefile | 10 +++++++++- deny.toml | 5 ++++- dev/release/README.md | 2 +- dev/release/create_rc.sh | 17 +---------------- dev/release/dependencies.sh | 3 ++- website/src/release.md | 2 +- 8 files changed, 30 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 571b46fcc9..2c181efd6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: # Keep versions in sync with the local install targets in Makefile. uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: - tool: taplo@0.9.3,cargo-machete@0.7.0 + tool: taplo@0.9.3,cargo-machete@0.7.0,cargo-deny@0.19.9 - name: Check License Header uses: apache/skywalking-eyes/header@a196742f472feaffafea537ce5a2a4c3c53a8de4 # v0.9.0 @@ -79,6 +79,9 @@ jobs: - name: Check crates package LICENSE and NOTICE run: make check-license-notice + - name: Check dependency licenses + run: dev/release/dependencies.sh check + # Trusted publishing cannot create a crate (https://crates.io/docs/trusted-publishing), # so a new publishable crate must be reserved on crates.io before it is merged. # See "Adding a new crate" in website/src/release.md. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d1a736ed16..3b75b10a96 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -118,6 +118,14 @@ tested in CI and developers have reproducible builds. In `Cargo.toml`, we specify the minimum version required to use iceberg-rust. This allows users to choose their dependency versions without always upgrading to the latest. +Dependency licenses must comply with the [ASF 3rd Party License Policy](https://www.apache.org/legal/resolved.html). +CI checks them against `deny.toml` with `cargo deny`; run `make check-dependency-licenses` to check locally. If a +license is rejected, find its category in the policy: + +- Category A: add the license to `allow`. +- Category B: add a per-crate entry to `exceptions`. +- Category X: not allowed by ASF policy; do not take the dependency. + ## Code of Conduct We expect all community members to follow our [Code of Conduct](https://www.apache.org/foundation/policies/conduct.html). diff --git a/Makefile b/Makefile index 5371c42287..bfb0c1da87 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,15 @@ check-public-api: install-cargo-public-api check-license-notice: bash ./dev/check_license_notice.sh -check: check-fmt check-clippy check-toml cargo-machete check-license-notice +# Keep version in sync with the CI lint install step in .github/workflows/ci.yml +# and EXPECTED_CARGO_DENY_VERSION in dev/release/dependencies.sh. +install-cargo-deny: + cargo install --locked cargo-deny@0.19.9 + +check-dependency-licenses: install-cargo-deny + bash ./dev/release/dependencies.sh check + +check: check-fmt check-clippy check-toml cargo-machete check-license-notice check-dependency-licenses doc-test: cargo test --no-fail-fast --doc --all-features --workspace diff --git a/deny.toml b/deny.toml index e8c3985e5d..27c3339ce3 100644 --- a/deny.toml +++ b/deny.toml @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +[graph] +# Include dependencies from optional features. +all-features = true + [licenses] allow = [ "Apache-2.0", @@ -26,7 +30,6 @@ allow = [ "CC0-1.0", "Zlib", "CDLA-Permissive-2.0", - "bzip2-1.0.6", # Category-A: https://issues.apache.org/jira/browse/LEGAL-660 "Unicode-3.0", # Boost Software License Version 1.0 is allowed (Category-A): diff --git a/dev/release/README.md b/dev/release/README.md index 24c014b0d1..93dfb4e06c 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -62,7 +62,7 @@ Defaults: `--check_deps 1` requires `cargo-deny`. Install it with: ```shell -cargo install --locked cargo-deny +make install-cargo-deny ``` `--check_publish 1` runs `cargo publish --workspace --dry-run`, which packages and compiles every crate and needs network access to crates.io. diff --git a/dev/release/create_rc.sh b/dev/release/create_rc.sh index 8970aab7b1..ae3a5207ee 100755 --- a/dev/release/create_rc.sh +++ b/dev/release/create_rc.sh @@ -157,16 +157,6 @@ require_checksum_command() { fi } -require_cargo_deny() { - require_command cargo - if ! cargo deny --version >/dev/null 2>&1; then - echo "This step requires 'cargo-deny' for dependency license checks." >&2 - echo "Install it with: cargo install --locked cargo-deny" >&2 - echo "To skip this step locally, pass: --check_deps 0" >&2 - return 1 - fi -} - require_gpg_secret_key() { require_command gpg if ! gpg --list-secret-keys --with-colons 2>/dev/null | grep -q '^sec'; then @@ -317,12 +307,7 @@ check_rc_tag_available() { } check_dependency_licenses() { - require_cargo_deny - ( - trap - ERR - cd "${REPO_ROOT}" - cargo deny check license - ) + "${SCRIPT_DIR}/dependencies.sh" check } # Packages and builds every crate as `cargo publish` would, without uploading, diff --git a/dev/release/dependencies.sh b/dev/release/dependencies.sh index 85e3b29dd2..e0d9e6a94b 100755 --- a/dev/release/dependencies.sh +++ b/dev/release/dependencies.sh @@ -25,7 +25,8 @@ if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 fi -# Keep this in sync with CARGO_DENY_VERSION in .github/workflows/dependencies.yml. +# Keep this in sync with the cargo-deny version in the CI lint install step in +# .github/workflows/ci.yml and the install-cargo-deny target in Makefile. # The generated DEPENDENCIES.rust.tsv files are version-sensitive, so the local # cargo-deny must match the version CI uses to avoid spurious diffs. EXPECTED_CARGO_DENY_VERSION="0.19.9" diff --git a/website/src/release.md b/website/src/release.md index 116e175645..5e8fcad2c3 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -64,7 +64,7 @@ The RC creation script requires a local GPG secret key when artifact signing or Install the release tooling used by the local scripts: -- `cargo-deny` +- `cargo-deny` (`make install-cargo-deny`) - `docker` - `gpg` - `svn` From af1da4c5bc86178c38c1c6db0578bdd9fb7021e3 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Sun, 4 Oct 2026 16:06:52 +0000 Subject: [PATCH 30/63] feat(storage-opendal)!: make the per-IO-operation timeout configurable (#3263) * feat(storage-opendal)!: make the per-IO-operation timeout configurable `iceberg-storage-opendal` wraps every FileIO operator in `TimeoutLayer::new()`, whose 10s `io_timeout` bounds each `read`/`write` and each method call on a returned reader or writer. Nothing in the property or builder surface can override it, so an operation that legitimately needs longer fails deterministically: the `RetryLayer` above re-sends the same request, which cannot fit in the budget either. Add `client.io-timeout-ms`, parsed into an `OpenDalClientConfig` built with `#[derive(Properties)]` and handed to `TimeoutLayer::with_io_timeout`. The config's fields are private, so later client settings such as retry are additive rather than breaking. The default is unchanged. BREAKING CHANGE: `OpenDalStorage` variants carry a `client` field, and `OpenDalStorage::Memory` is now a struct variant. * refactor(storage-opendal): address review feedback on the IO timeout Quote the rejected value in the parse error, so an empty input shows as `value: ""`. `TimeoutLayer` exposes no getters, so a new test compares its `Debug` output to pin `DEFAULT_IO_TIMEOUT_MS` to OpenDAL's default. The parsing test now uses the constant instead of repeating the literal. Add a serde round-trip test for the `client` field, including a payload that omits it. Drop the `DEFAULT` associated const, which only fed a fallback arm that cannot run. Update the upstream `test_writer_close_returns_stored_size` for the struct variant, and reuse `StorageConfig::with_prop` and `empty_resolving_storage` in the tests. * refactor(storage-opendal): rename the IO timeout key to opendal.io-timeout-ms Only `iceberg-storage-opendal` honors the key, so namespace it under `opendal.` and define it as `OPENDAL_IO_TIMEOUT_MS` next to `OpenDalClientConfig`, rather than in `iceberg::io`. This matches how `ADLS_SAS_TOKEN` and `GCS_ALLOW_ANONYMOUS` sit next to the configs that parse them. With the key moved, the `iceberg` crate needs no change. * refactor(storage-opendal): rename the client field to client_config `client` reads like an HTTP client handle. Rename the field on every `OpenDalStorage` variant, and the `OpenDalStorage::client()` accessor, to `client_config` to match `OpenDalClientConfig`. `opendal_config` would be ambiguous next to the backend `config` field, which holds an OpenDAL config too. `test_default_io_timeout_matches_opendal` compares `Debug` output. Both sides go through the same derived impl, so a format change cannot make them differ. A `Debug` that stopped printing `io_timeout` would make the check pass vacuously, so also assert that a different `io_timeout` produces a different string. * fix(storage-opendal): reject a zero IO timeout when deserializing `parse_io_timeout_ms` rejected zero, but the derived `Deserialize` did not go through it. A serialized `OpenDalStorage` carrying `"io_timeout_ms": 0` was rebuilt without error, and the zero reached `TimeoutLayer`. Store the timeout as a `NonZeroU64`. `str::parse` and serde's `NonZeroU64` impl both reject zero, so both construction paths share one check. The serialized form is still a plain integer. The generated getter now returns `&NonZeroU64`, because the `Properties` derive only returns by value for the primitive types it recognizes. The parsing test also covers the edges of the valid range. `1` and `u64::MAX` are accepted, and `u64::MAX + 1` is rejected. * refactor(storage-opendal): return the IO timeout as a Duration `OpenDalClientConfig::io_timeout_ms()` returned `&NonZeroU64`, because the `Properties` derive only returns primitives by value. Replace the generated getter with `io_timeout()`, which returns a `Duration`, and keep the millisecond field private. Expose the default as `OPENDAL_IO_TIMEOUT_MS_DEFAULT`, following the `*_DEFAULT` consts in `iceberg`, so the key's doc links to it instead of repeating the number. Keep the `ParseIntError` as the source of a rejected value, as `parse_pool_property` does, so the message says why it failed. Document that the serialized form of `OpenDalStorage` is not stable across crate versions, as `FileIO::serialize_all` already does for its own. Test that the configured timeout reaches `TimeoutLayer`. A memory operator behind `ConcurrentLimitLayer::new(0)` stalls every IO call, and paused tokio time skips the wait and the retry backoff. Check propagation through `OpenDalResolvingStorage` for every enabled scheme, not only S3. * refactor(storage-opendal): take the client config defaults from the derive Build `OpenDalClientConfig::default()` from `from_properties` with no properties, so each default is declared once in `#[property]` and serde and `from_properties` cannot disagree as settings are added. Say in the `parse_io_timeout_ms` doc that it exists to quote the value and name the unit, and that `NonZeroU64` is what rejects zero. Fold the valid-value asserts in the parsing test into one loop. * refactor(storage-opendal): build the client config default directly `Default` delegated to `from_properties` with no properties and called `expect`. A later setting without a `#[property]` default would make it panic, and because of the container-level `#[serde(default)]`, serde calls `default()` on every `client_config` it deserializes. Build the struct directly, and keep the two defaults equal with `test_default_matches_property_defaults`. Name OpenDAL's 60-second control-operation budget in the `OPENDAL_IO_TIMEOUT_MS` doc, along with the calls it covers here, `exists` and `metadata`. `test_default_timeouts_match_opendal` now pins both defaults that the doc states. Pin the serialized-form break with a test that rejects the old `"LocalFs"`, `"Memory"` and `{"Memory": null}` forms. Match the stall test's error on `{ timeout: 45 }`, so a 450-second budget cannot pass. * refactor(storage-opendal): keep the client config accessors crate-private `OpenDalStorage::client_config()`, `OpenDalClientConfig::io_timeout()` and `OPENDAL_IO_TIMEOUT_MS_DEFAULT` have no callers outside the crate. Keep them out of the public API so they can be exposed later without a breaking change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * refactor(storage-opendal): trim redundant docs, comments and test Fold the `u64` default into the `NonZeroU64` const, drop `test_default_matches_property_defaults` (one field, one shared const), shorten docs and comments that restated the code, and keep the original `op` binding in the `Memory` arm of `create_operator`. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs(storage-opendal): clarify the IO timeout and serialization docs Name OpenDAL's `io_timeout` and `timeout` settings so it is clear the property bounds each IO call per retry attempt and leaves control operations such as `stat` at 60 seconds. Align the `OpenDalStorage` serialization note with the wording on `FileIO::serialize_all`. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * test(storage-opendal): check OpenDalStorageFactory propagates the IO timeout Every `OpenDalStorageFactory::build` arm could fall back to the default `OpenDalClientConfig` without any test failing. Build each enabled backend with a 45000 ms timeout and read it back from the serialized storage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: ovoievodin Co-authored-by: Kevin Liu Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 2 + crates/storage/opendal/Cargo.toml | 4 +- crates/storage/opendal/public-api.txt | 24 +- crates/storage/opendal/src/lib.rs | 330 ++++++++++++++++++++++-- crates/storage/opendal/src/resolving.rs | 51 +++- 5 files changed, 387 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fcf5dd4497..7b2115a713 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2935,12 +2935,14 @@ dependencies = [ "cfg-if 1.0.4", "futures", "iceberg", + "iceberg-property-macro", "iceberg_test_utils", "opendal", "reqsign-aws-v4", "reqsign-core", "reqwest 0.12.28", "serde", + "serde_json", "tempfile", "tokio", "typetag", diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index c6ea9f9b67..d3403598e8 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -54,6 +54,7 @@ bytes = { workspace = true } cfg-if = { workspace = true } futures = { workspace = true } iceberg = { workspace = true } +iceberg-property-macro = { workspace = true } opendal = { workspace = true } reqsign-aws-v4 = { version = "3.0.0", optional = true } reqsign-core = { version = "3.0.0", optional = true } @@ -65,8 +66,9 @@ url = { workspace = true } async-trait = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } reqwest = { workspace = true } +serde_json = { workspace = true } tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros"] } +tokio = { workspace = true, features = ["macros", "test-util"] } [lints] workspace = true diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index d8c4ecdb38..c2e310335c 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -3,16 +3,24 @@ pub use iceberg_storage_opendal::AwsCredential pub use iceberg_storage_opendal::ProvideCredential pub enum iceberg_storage_opendal::OpenDalStorage pub iceberg_storage_opendal::OpenDalStorage::Azdls +pub iceberg_storage_opendal::OpenDalStorage::Azdls::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::Gcs +pub iceberg_storage_opendal::OpenDalStorage::Gcs::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::Hf +pub iceberg_storage_opendal::OpenDalStorage::Hf::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::LocalFs -pub iceberg_storage_opendal::OpenDalStorage::Memory(opendal_core::types::operator::operator::Operator) +pub iceberg_storage_opendal::OpenDalStorage::LocalFs::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::Memory +pub iceberg_storage_opendal::OpenDalStorage::Memory::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::Memory::operator: opendal_core::types::operator::operator::Operator pub iceberg_storage_opendal::OpenDalStorage::Oss +pub iceberg_storage_opendal::OpenDalStorage::Oss::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Oss::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::S3 +pub iceberg_storage_opendal::OpenDalStorage::S3::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::S3::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::S3::customized_credential_load: core::option::Option impl core::clone::Clone for iceberg_storage_opendal::OpenDalStorage @@ -61,6 +69,19 @@ impl core::clone::Clone for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::clone(&self) -> Self impl core::fmt::Debug for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_storage_opendal::OpenDalClientConfig +impl iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::from_properties(properties: &std::collections::hash::map::HashMap) -> iceberg::error::Result +impl core::clone::Clone for iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::clone(&self) -> iceberg_storage_opendal::OpenDalClientConfig +impl core::default::Default for iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::default() -> Self +impl core::fmt::Debug for iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalClientConfig where iceberg_storage_opendal::OpenDalClientConfig: core::default::Default +pub fn iceberg_storage_opendal::OpenDalClientConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg_storage_opendal::OpenDalResolvingStorage impl core::fmt::Debug for iceberg_storage_opendal::OpenDalResolvingStorage pub fn iceberg_storage_opendal::OpenDalResolvingStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -96,3 +117,4 @@ impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalResolvingSto pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalResolvingStorageFactory pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub const iceberg_storage_opendal::OPENDAL_IO_TIMEOUT_MS: &str diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index fe485cb915..f58d6f534c 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -26,7 +26,9 @@ mod utils; use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::num::NonZeroU64; use std::sync::Arc; +use std::time::Duration; use async_trait::async_trait; use bytes::Bytes; @@ -38,6 +40,7 @@ use iceberg::io::{ StorageFactory, }; use iceberg::{Error, ErrorKind, Result}; +use iceberg_property_macro::Properties; use opendal::Operator; use opendal::layers::{RetryLayer, TimeoutLayer}; use serde::{Deserialize, Serialize}; @@ -100,6 +103,54 @@ cfg_if! { mod resolving; pub use resolving::{OpenDalResolvingStorage, OpenDalResolvingStorageFactory}; +/// Timeout in milliseconds for each IO call on a reader, writer, lister or deleter, applied per +/// retry attempt via OpenDAL's `TimeoutLayer::with_io_timeout`. Defaults to 10 seconds. +/// +/// OpenDAL's separate `timeout`, which bounds whole control operations such as `stat`, is not +/// affected and stays at its 60-second default. +pub const OPENDAL_IO_TIMEOUT_MS: &str = "opendal.io-timeout-ms"; + +/// Matches OpenDAL's `TimeoutLayer` default. +const DEFAULT_IO_TIMEOUT_MS: NonZeroU64 = NonZeroU64::new(10_000).unwrap(); + +/// Backend-independent client settings shared by every [`OpenDalStorage`] variant. +#[derive(Clone, Debug, Properties, Serialize, Deserialize)] +#[serde(default)] +pub struct OpenDalClientConfig { + /// IO timeout in milliseconds. + #[property( + key = OPENDAL_IO_TIMEOUT_MS, + default = DEFAULT_IO_TIMEOUT_MS, + parse_with = parse_io_timeout_ms + )] + io_timeout_ms: NonZeroU64, +} + +impl Default for OpenDalClientConfig { + fn default() -> Self { + Self { + io_timeout_ms: DEFAULT_IO_TIMEOUT_MS, + } + } +} + +impl OpenDalClientConfig { + pub(crate) fn io_timeout(&self) -> Duration { + Duration::from_millis(self.io_timeout_ms.get()) + } +} + +fn parse_io_timeout_ms(value: &str) -> Result { + value.parse().map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + "Expected a positive integer number of milliseconds", + ) + .with_context("value", format!("{value:?}")) + .with_source(error) + }) +} + /// OpenDAL-based storage factory. /// /// Maps scheme to the corresponding OpenDalStorage storage variant. @@ -163,35 +214,42 @@ where impl StorageFactory for OpenDalStorageFactory { #[allow(unused_variables)] fn build(&self, config: &StorageConfig) -> Result> { + let client_config = OpenDalClientConfig::from_properties(config.props())?; match self { #[cfg(feature = "opendal-memory")] - OpenDalStorageFactory::Memory => { - Ok(Arc::new(OpenDalStorage::Memory(memory_config_build()?))) - } + OpenDalStorageFactory::Memory => Ok(Arc::new(OpenDalStorage::Memory { + operator: memory_config_build()?, + client_config, + })), #[cfg(feature = "opendal-fs")] - OpenDalStorageFactory::Fs => Ok(Arc::new(OpenDalStorage::LocalFs)), + OpenDalStorageFactory::Fs => Ok(Arc::new(OpenDalStorage::LocalFs { client_config })), #[cfg(feature = "opendal-s3")] OpenDalStorageFactory::S3 { customized_credential_load, } => Ok(Arc::new(OpenDalStorage::S3 { config: s3_config_parse(config.props().clone())?.into(), customized_credential_load: customized_credential_load.clone(), + client_config, })), #[cfg(feature = "opendal-gcs")] OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs { config: gcs_config_parse(config.props().clone())?.into(), + client_config, })), #[cfg(feature = "opendal-oss")] OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss { config: oss_config_parse(config.props().clone())?.into(), + client_config, })), #[cfg(feature = "opendal-azdls")] OpenDalStorageFactory::Azdls => Ok(Arc::new(OpenDalStorage::Azdls { config: azdls_config_parse(config.props().clone())?.into(), + client_config, })), #[cfg(feature = "opendal-hf")] OpenDalStorageFactory::Hf => Ok(Arc::new(OpenDalStorage::Hf { config: hf_config_parse(config.props().clone())?.into(), + client_config, })), #[cfg(all( not(feature = "opendal-memory"), @@ -217,14 +275,27 @@ fn default_memory_operator() -> Operator { } /// OpenDAL-based storage implementation. +/// +/// The serialized representation is not a stable format and may change between crate versions. #[derive(Clone, Debug, Serialize, Deserialize)] pub enum OpenDalStorage { /// Memory storage variant. #[cfg(feature = "opendal-memory")] - Memory(#[serde(skip, default = "self::default_memory_operator")] Operator), + Memory { + /// Pre-built memory operator. + #[serde(skip, default = "self::default_memory_operator")] + operator: Operator, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, /// Local filesystem storage variant. #[cfg(feature = "opendal-fs")] - LocalFs, + LocalFs { + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, /// S3 storage variant. /// /// Accepts any S3-family URL (`s3://`, `s3a://`, `s3n://`); the scheme is @@ -236,18 +307,27 @@ pub enum OpenDalStorage { /// Custom AWS credential loader. #[serde(skip)] customized_credential_load: Option, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, }, /// GCS storage variant. #[cfg(feature = "opendal-gcs")] Gcs { /// GCS configuration. config: Arc, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, }, /// OSS storage variant. #[cfg(feature = "opendal-oss")] Oss { /// OSS configuration. config: Arc, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, }, /// Azure Data Lake Storage variant. /// @@ -259,6 +339,9 @@ pub enum OpenDalStorage { Azdls { /// Azure DLS configuration. config: Arc, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, }, /// HuggingFace Hub storage variant. /// @@ -269,6 +352,9 @@ pub enum OpenDalStorage { Hf { /// HuggingFace Hub configuration (token + endpoint). config: Arc, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, }, } @@ -293,7 +379,7 @@ impl OpenDalStorage { let path = path.as_ref(); let (operator, relative_path): (Operator, &str) = match self { #[cfg(feature = "opendal-memory")] - OpenDalStorage::Memory(op) => { + OpenDalStorage::Memory { operator: op, .. } => { if let Some(stripped) = path.strip_prefix("memory:/") { (op.clone(), stripped) } else { @@ -301,7 +387,7 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-fs")] - OpenDalStorage::LocalFs => { + OpenDalStorage::LocalFs { .. } => { let op = fs_config_build()?; if let Some(stripped) = path.strip_prefix("file:/") { (op, stripped) @@ -313,6 +399,7 @@ impl OpenDalStorage { OpenDalStorage::S3 { config, customized_credential_load, + .. } => { let op = s3_config_build(config, customized_credential_load, path)?; let op_info = op.info(); @@ -336,7 +423,7 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-gcs")] - OpenDalStorage::Gcs { config } => { + OpenDalStorage::Gcs { config, .. } => { let operator = gcs_config_build(config, path)?; let prefix = format!("gs://{}/", operator.info().name()); if path.starts_with(&prefix) { @@ -349,7 +436,7 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-oss")] - OpenDalStorage::Oss { config } => { + OpenDalStorage::Oss { config, .. } => { let op = oss_config_build(config, path)?; let prefix = format!("oss://{}/", op.info().name()); if path.starts_with(&prefix) { @@ -362,9 +449,9 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-azdls")] - OpenDalStorage::Azdls { config } => azdls_create_operator(path, config)?, + OpenDalStorage::Azdls { config, .. } => azdls_create_operator(path, config)?, #[cfg(feature = "opendal-hf")] - OpenDalStorage::Hf { config } => hf_config_build(config, path)?, + OpenDalStorage::Hf { config, .. } => hf_config_build(config, path)?, #[cfg(all( not(feature = "opendal-s3"), not(feature = "opendal-fs"), @@ -391,10 +478,41 @@ impl OpenDalStorage { // Transient errors are common for object stores; we retry temporary // failures with exponential backoff. The retry behavior also // benefits non-object-store backends. - let operator = operator.layer(TimeoutLayer::new()).layer(RetryLayer::new()); + let operator = operator + .layer(TimeoutLayer::new().with_io_timeout(self.client_config().io_timeout())) + .layer(RetryLayer::new()); Ok((operator, relative_path)) } + pub(crate) fn client_config(&self) -> &OpenDalClientConfig { + match self { + #[cfg(feature = "opendal-memory")] + OpenDalStorage::Memory { client_config, .. } => client_config, + #[cfg(feature = "opendal-fs")] + OpenDalStorage::LocalFs { client_config } => client_config, + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { client_config, .. } => client_config, + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { client_config, .. } => client_config, + #[cfg(feature = "opendal-oss")] + OpenDalStorage::Oss { client_config, .. } => client_config, + #[cfg(feature = "opendal-azdls")] + OpenDalStorage::Azdls { client_config, .. } => client_config, + #[cfg(feature = "opendal-hf")] + OpenDalStorage::Hf { client_config, .. } => client_config, + #[cfg(all( + not(feature = "opendal-memory"), + not(feature = "opendal-s3"), + not(feature = "opendal-fs"), + not(feature = "opendal-gcs"), + not(feature = "opendal-oss"), + not(feature = "opendal-azdls"), + not(feature = "opendal-hf"), + ))] + _ => unreachable!(), + } + } + /// Returns a cache key used by `delete_stream` to group paths by storage operator. /// /// For most backends the URL host (bucket name) is sufficient. For HF the host @@ -419,9 +537,11 @@ impl OpenDalStorage { pub(crate) fn relativize_path<'a>(&self, path: &'a str) -> Result<&'a str> { match self { #[cfg(feature = "opendal-memory")] - OpenDalStorage::Memory(_) => Ok(path.strip_prefix("memory:/").unwrap_or(&path[1..])), + OpenDalStorage::Memory { .. } => { + Ok(path.strip_prefix("memory:/").unwrap_or(&path[1..])) + } #[cfg(feature = "opendal-fs")] - OpenDalStorage::LocalFs => Ok(path.strip_prefix("file:/").unwrap_or(&path[1..])), + OpenDalStorage::LocalFs { .. } => Ok(path.strip_prefix("file:/").unwrap_or(&path[1..])), #[cfg(feature = "opendal-s3")] OpenDalStorage::S3 { .. } => { let url = url::Url::parse(path)?; @@ -480,7 +600,7 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-azdls")] - OpenDalStorage::Azdls { config } => { + OpenDalStorage::Azdls { config, .. } => { let azure_path = path.parse::()?; match_path_with_config(&azure_path, config)?; let relative_path_len = azure_path.path.len(); @@ -687,6 +807,164 @@ impl FileWrite for OpenDalWriter { mod tests { use super::*; + fn client_config(value: &str) -> Result { + OpenDalClientConfig::from_properties(&HashMap::from([( + OPENDAL_IO_TIMEOUT_MS.to_string(), + value.to_string(), + )])) + } + + #[test] + fn test_io_timeout_parsing() { + let unset = OpenDalClientConfig::from_properties(&HashMap::new()).unwrap(); + assert_eq!( + unset.io_timeout(), + Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get()) + ); + + let max = u64::MAX.to_string(); + for (valid, ms) in [("45000", 45_000), ("1", 1), (max.as_str(), u64::MAX)] { + assert_eq!( + client_config(valid).unwrap().io_timeout(), + Duration::from_millis(ms), + "{valid}" + ); + } + + for invalid in ["0", "-1", "12.5", "abc", "", "18446744073709551616"] { + let err = client_config(invalid).unwrap_err().to_string(); + assert!(err.contains(OPENDAL_IO_TIMEOUT_MS), "{invalid}"); + assert!(err.contains(&format!("value: {invalid:?}")), "{err}"); + let reason = invalid.parse::().unwrap_err().to_string(); + assert!(err.contains(&reason), "{err}"); + } + } + + #[test] + fn test_default_timeouts_match_opendal() { + // `TimeoutLayer` has no getters, so compare through `Debug`. + let opendal_default = format!("{:?}", TimeoutLayer::new()); + let layer = |timeout, io_timeout| { + format!( + "{:?}", + TimeoutLayer::new() + .with_timeout(timeout) + .with_io_timeout(io_timeout) + ) + }; + let control = Duration::from_secs(60); + let io = Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get()); + assert_eq!(opendal_default, layer(control, io)); + assert_ne!(opendal_default, layer(control + Duration::from_secs(1), io)); + assert_ne!( + opendal_default, + layer(control, io + Duration::from_millis(1)) + ); + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_client_config_serde_round_trip() { + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: None, + client_config: client_config("45000").unwrap(), + }; + + let mut value = serde_json::to_value(&storage).unwrap(); + let restored: OpenDalStorage = serde_json::from_value(value.clone()).unwrap(); + assert_eq!( + restored.client_config().io_timeout(), + Duration::from_secs(45) + ); + + let mut zero = value.clone(); + zero["S3"]["client_config"]["io_timeout_ms"] = 0.into(); + assert!(serde_json::from_value::(zero).is_err()); + + value["S3"].as_object_mut().unwrap().remove("client_config"); + let restored: OpenDalStorage = serde_json::from_value(value).unwrap(); + assert_eq!( + restored.client_config().io_timeout(), + Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get()) + ); + } + + #[cfg(all(feature = "opendal-fs", feature = "opendal-memory"))] + #[test] + fn test_old_unit_variant_forms_are_rejected() { + for old in [r#""LocalFs""#, r#""Memory""#, r#"{"Memory":null}"#] { + assert!( + serde_json::from_str::(old).is_err(), + "{old}" + ); + } + } + + #[cfg(feature = "opendal-memory")] + #[test] + fn test_factory_rejects_invalid_io_timeout() { + let config = StorageConfig::new().with_prop(OPENDAL_IO_TIMEOUT_MS, "nope"); + + let err = OpenDalStorageFactory::Memory.build(&config).unwrap_err(); + assert!(err.to_string().contains(OPENDAL_IO_TIMEOUT_MS)); + } + + #[test] + fn test_factory_propagates_io_timeout() { + let config = StorageConfig::new().with_prop(OPENDAL_IO_TIMEOUT_MS, "45000"); + let factories = [ + #[cfg(feature = "opendal-memory")] + OpenDalStorageFactory::Memory, + #[cfg(feature = "opendal-fs")] + OpenDalStorageFactory::Fs, + #[cfg(feature = "opendal-s3")] + OpenDalStorageFactory::S3 { + customized_credential_load: None, + }, + #[cfg(feature = "opendal-gcs")] + OpenDalStorageFactory::Gcs, + #[cfg(feature = "opendal-oss")] + OpenDalStorageFactory::Oss, + #[cfg(feature = "opendal-azdls")] + OpenDalStorageFactory::Azdls, + #[cfg(feature = "opendal-hf")] + OpenDalStorageFactory::Hf, + ]; + for factory in factories { + // `build` returns `dyn Storage`, so read the config back from its serialized form. + let storage = factory.build(&config).unwrap(); + let json = serde_json::to_value(&*storage).unwrap(); + let client_config = json + .as_object() + .unwrap() + .values() + .find_map(|variant| variant.get("client_config")) + .unwrap_or_else(|| panic!("{json}")); + assert_eq!(client_config["io_timeout_ms"], 45_000, "{factory:?}"); + } + } + + #[cfg(feature = "opendal-memory")] + #[tokio::test(start_paused = true)] + async fn test_io_timeout_reaches_timeout_layer() { + use opendal::layers::ConcurrentLimitLayer; + + // A zero concurrency limit stalls every IO call; paused time skips the waits. + let storage = OpenDalStorage::Memory { + operator: default_memory_operator().layer(ConcurrentLimitLayer::new(0)), + client_config: client_config("45000").unwrap(), + }; + + let err = storage + .read("memory:/stalled") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("io operation timeout reached"), "{err}"); + assert!(err.contains("{ timeout: 45 }"), "{err}"); + } + #[cfg(feature = "opendal-s3")] #[derive(Debug)] struct EmptyCredentialLoader; @@ -732,7 +1010,10 @@ mod tests { // Note: the memory service does report a content length, so this only pins the happy // path. The counter in `OpenDalWriter` is what covers services that don't, such as S3. - let storage = Arc::new(OpenDalStorage::Memory(default_memory_operator())); + let storage = Arc::new(OpenDalStorage::Memory { + operator: default_memory_operator(), + client_config: OpenDalClientConfig::default(), + }); let path = "memory:///stored-size"; for plaintext in [ Bytes::new(), @@ -760,7 +1041,10 @@ mod tests { #[cfg(feature = "opendal-memory")] #[test] fn test_relativize_path_memory() { - let storage = OpenDalStorage::Memory(default_memory_operator()); + let storage = OpenDalStorage::Memory { + operator: default_memory_operator(), + client_config: OpenDalClientConfig::default(), + }; assert_eq!( storage.relativize_path("memory:/path/to/file").unwrap(), @@ -776,7 +1060,9 @@ mod tests { #[cfg(feature = "opendal-fs")] #[test] fn test_relativize_path_fs() { - let storage = OpenDalStorage::LocalFs; + let storage = OpenDalStorage::LocalFs { + client_config: OpenDalClientConfig::default(), + }; assert_eq!( storage @@ -796,6 +1082,7 @@ mod tests { let storage = OpenDalStorage::S3 { config: Arc::new(S3Config::default()), customized_credential_load: None, + client_config: OpenDalClientConfig::default(), }; // All S3-family schemes are accepted by the same storage instance. @@ -816,6 +1103,7 @@ mod tests { fn test_relativize_path_gcs() { let storage = OpenDalStorage::Gcs { config: Arc::new(GcsConfig::default()), + client_config: OpenDalClientConfig::default(), }; assert_eq!( @@ -831,6 +1119,7 @@ mod tests { fn test_relativize_path_gcs_invalid_scheme() { let storage = OpenDalStorage::Gcs { config: Arc::new(GcsConfig::default()), + client_config: OpenDalClientConfig::default(), }; assert!( @@ -845,6 +1134,7 @@ mod tests { fn test_relativize_path_oss() { let storage = OpenDalStorage::Oss { config: Arc::new(OssConfig::default()), + client_config: OpenDalClientConfig::default(), }; assert_eq!( @@ -860,6 +1150,7 @@ mod tests { fn test_relativize_path_oss_invalid_scheme() { let storage = OpenDalStorage::Oss { config: Arc::new(OssConfig::default()), + client_config: OpenDalClientConfig::default(), }; assert!( @@ -878,6 +1169,7 @@ mod tests { endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), ..Default::default() }), + client_config: OpenDalClientConfig::default(), }; assert_eq!( diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 3b99b08fc0..51e76da472 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -33,9 +33,9 @@ use iceberg::{Error, ErrorKind, Result}; use serde::{Deserialize, Serialize}; use url::Url; -use crate::OpenDalStorage; #[cfg(feature = "opendal-s3")] use crate::s3::CustomAwsCredentialLoader; +use crate::{OpenDalClientConfig, OpenDalStorage}; /// Schemes supported by OpenDalResolvingStorage pub const SCHEME_MEMORY: &str = "memory"; @@ -86,6 +86,7 @@ fn build_storage_for_scheme( props: &HashMap, #[cfg(feature = "opendal-s3")] customized_credential_load: &Option, ) -> Result { + let client_config = OpenDalClientConfig::from_properties(props)?; match scheme { #[cfg(feature = "opendal-s3")] "s3" => { @@ -93,6 +94,7 @@ fn build_storage_for_scheme( Ok(OpenDalStorage::S3 { config: Arc::new(config), customized_credential_load: customized_credential_load.clone(), + client_config, }) } #[cfg(feature = "opendal-gcs")] @@ -100,6 +102,7 @@ fn build_storage_for_scheme( let config = crate::gcs::gcs_config_parse(props.clone())?; Ok(OpenDalStorage::Gcs { config: Arc::new(config), + client_config, }) } #[cfg(feature = "opendal-oss")] @@ -107,6 +110,7 @@ fn build_storage_for_scheme( let config = crate::oss::oss_config_parse(props.clone())?; Ok(OpenDalStorage::Oss { config: Arc::new(config), + client_config, }) } #[cfg(feature = "opendal-azdls")] @@ -114,17 +118,22 @@ fn build_storage_for_scheme( let config = crate::azdls::azdls_config_parse(props.clone())?; Ok(OpenDalStorage::Azdls { config: Arc::new(config), + client_config, }) } #[cfg(feature = "opendal-fs")] - "file" => Ok(OpenDalStorage::LocalFs), + "file" => Ok(OpenDalStorage::LocalFs { client_config }), #[cfg(feature = "opendal-memory")] - "memory" => Ok(OpenDalStorage::Memory(crate::memory::memory_config_build()?)), + "memory" => Ok(OpenDalStorage::Memory { + operator: crate::memory::memory_config_build()?, + client_config, + }), #[cfg(feature = "opendal-hf")] "hf" => { let config = crate::hf::hf_config_parse(props.clone())?; Ok(OpenDalStorage::Hf { config: Arc::new(config), + client_config, }) } unsupported => Err(Error::new( @@ -334,7 +343,10 @@ impl Storage for OpenDalResolvingStorage { #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; + use crate::OPENDAL_IO_TIMEOUT_MS; #[cfg(feature = "opendal-s3")] #[derive(Debug)] @@ -377,6 +389,39 @@ mod tests { } } + #[test] + fn test_resolve_propagates_io_timeout() { + let mut storage = empty_resolving_storage(); + storage + .props + .insert(OPENDAL_IO_TIMEOUT_MS.to_string(), "45000".to_string()); + + let paths: &[&str] = &[ + #[cfg(feature = "opendal-memory")] + "memory:/key", + #[cfg(feature = "opendal-fs")] + "file:/key", + #[cfg(feature = "opendal-s3")] + "s3://bucket/key", + #[cfg(feature = "opendal-gcs")] + "gs://bucket/key", + #[cfg(feature = "opendal-oss")] + "oss://bucket/key", + #[cfg(feature = "opendal-azdls")] + "abfss://myfs@myaccount.dfs.core.windows.net/key", + #[cfg(feature = "opendal-hf")] + "hf://datasets/user/repo/key", + ]; + for path in paths { + let resolved = storage.resolve(path).unwrap(); + assert_eq!( + resolved.client_config().io_timeout(), + Duration::from_secs(45), + "{path}" + ); + } + } + #[cfg(feature = "opendal-s3")] #[test] fn test_resolve_s3_aliases_share_instance() { From ecba0b537a1f78ec8125c1a26f6dfab3f95b509e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 4 Oct 2026 21:10:38 +0000 Subject: [PATCH 31/63] fix(transform): floor day transform for pre-epoch timestamps (#3323) * fix(transform): floor day transform for pre-epoch timestamps `Day::day_timestamp_micro` and `Day::day_timestamp_nano` took the whole seconds of a timestamp with `/`, which truncates toward zero, while the fraction of the second came from `rem_euclid`, which floors. For a negative timestamp that is not on a whole second the seconds came out one too high, so a timestamp in the last second of a day before 1969-12-31 got the next day. Compute the day with `div_euclid` on the length of a day instead, the way `Hour` already does. This gives the same days as Iceberg Java's `DateTimeUtil`, except for a pre-epoch timestamp exactly 999999 microseconds (999999999 nanoseconds) after midnight, which Java puts in the day before. iceberg-rust already returned the calendar day for those, as PyIceberg and the `year`, `month` and `hour` transforms do, and following Java there would make the transform non-monotonic. Files written before this change can hold those rows one day too high. Inclusive projection already widens pre-epoch `day` predicates by one day to cover such files, as Java does, but only for `timestamp` columns. Apply it to every timestamp type. * refactor(transform): list timestamp types for day projection adjustment Match how transform.rs lists the timestamp types elsewhere, and keep the adjustment opt-in if `day` ever gains another source type. No behavior change: `day` accepts `date` and the four timestamp types. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * chore(transform): trim day transform comments Drop comments that narrate history or restate the test code. The history is in the PR description. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Kevin Liu Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/iceberg/src/spec/transform.rs | 8 +- crates/iceberg/src/transform/temporal.rs | 153 +++++++++++++++-------- 2 files changed, 105 insertions(+), 56 deletions(-) diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index c4c915c080..d87054ab73 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -873,7 +873,13 @@ impl Transform { transformed: &Datum, ) -> Option { let should_adjust = match self { - Transform::Day => matches!(original.data_type(), PrimitiveType::Timestamp), + Transform::Day => matches!( + original.data_type(), + PrimitiveType::Timestamp + | PrimitiveType::Timestamptz + | PrimitiveType::TimestampNs + | PrimitiveType::TimestamptzNs + ), Transform::Year | Transform::Month => true, _ => false, }; diff --git a/crates/iceberg/src/transform/temporal.rs b/crates/iceberg/src/transform/temporal.rs index 04a8256f53..02ac2b4b93 100644 --- a/crates/iceberg/src/transform/temporal.rs +++ b/crates/iceberg/src/transform/temporal.rs @@ -24,7 +24,7 @@ use arrow_array::{ Array, ArrayRef, Date32Array, Int32Array, TimestampMicrosecondArray, TimestampNanosecondArray, }; use arrow_schema::{DataType, TimeUnit}; -use chrono::{DateTime, Datelike, Duration}; +use chrono::{DateTime, Datelike}; use super::TransformFunction; use crate::error::invalid_data; @@ -37,10 +37,10 @@ const MICROSECONDS_PER_HOUR: i64 = 3_600_000_000; const NANOSECONDS_PER_HOUR: i64 = 3_600_000_000_000; /// Year of unix epoch. const UNIX_EPOCH_YEAR: i32 = 1970; -/// One second in micros. -const MICROS_PER_SECOND: i64 = 1_000_000; -/// One second in nanos. -const NANOS_PER_SECOND: i64 = 1_000_000_000; +/// Microseconds in one day. +const MICROSECONDS_PER_DAY: i64 = 86_400_000_000; +/// Nanoseconds in one day. +const NANOSECONDS_PER_DAY: i64 = 86_400_000_000_000; /// Extract a date or timestamp year, as years from 1970 #[derive(Debug)] @@ -205,50 +205,13 @@ pub struct Day; impl Day { #[inline] - fn day_timestamp_micro(v: i64) -> Result { - let secs = v / MICROS_PER_SECOND; - - let (nanos, offset) = if v >= 0 { - let nanos = (v.rem_euclid(MICROS_PER_SECOND) * 1_000) as u32; - let offset = 0i64; - (nanos, offset) - } else { - let v = v + 1; - let nanos = (v.rem_euclid(MICROS_PER_SECOND) * 1_000) as u32; - let offset = 1i64; - (nanos, offset) - }; - - let delta = Duration::new(secs, nanos).ok_or_else(|| { - invalid_data!("Failed to create 'TimeDelta' from seconds {secs} and nanos {nanos}") - })?; - - let days = (delta.num_days() - offset) as i32; - - Ok(days) + fn day_timestamp_micro(v: i64) -> i32 { + v.div_euclid(MICROSECONDS_PER_DAY) as i32 } - fn day_timestamp_nano(v: i64) -> Result { - let secs = v / NANOS_PER_SECOND; - - let (nanos, offset) = if v >= 0 { - let nanos = (v.rem_euclid(NANOS_PER_SECOND)) as u32; - let offset = 0i64; - (nanos, offset) - } else { - let v = v + 1; - let nanos = (v.rem_euclid(NANOS_PER_SECOND)) as u32; - let offset = 1i64; - (nanos, offset) - }; - - let delta = Duration::new(secs, nanos).ok_or_else(|| { - invalid_data!("Failed to create 'TimeDelta' from seconds {secs} and nanos {nanos}") - })?; - - let days = (delta.num_days() - offset) as i32; - - Ok(days) + #[inline] + fn day_timestamp_nano(v: i64) -> i32 { + v.div_euclid(NANOSECONDS_PER_DAY) as i32 } } @@ -259,12 +222,12 @@ impl TransformFunction for Day { .as_any() .downcast_ref::() .unwrap() - .try_unary(|v| -> Result { Self::day_timestamp_micro(v) })?, + .unary(|v| -> i32 { Self::day_timestamp_micro(v) }), DataType::Timestamp(TimeUnit::Nanosecond, _) => input .as_any() .downcast_ref::() .unwrap() - .try_unary(|v| -> Result { Self::day_timestamp_nano(v) })?, + .unary(|v| -> i32 { Self::day_timestamp_nano(v) }), DataType::Date32 => input .as_any() .downcast_ref::() @@ -286,15 +249,13 @@ impl TransformFunction for Day { fn transform_literal(&self, input: &Datum) -> Result> { let val = match (input.data_type(), input.literal()) { (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => *v, - (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => Self::day_timestamp_micro(*v)?, + (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => Self::day_timestamp_micro(*v), (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(v)) => { - Self::day_timestamp_micro(*v)? - } - (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(v)) => { - Self::day_timestamp_nano(*v)? + Self::day_timestamp_micro(*v) } + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(v)) => Self::day_timestamp_nano(*v), (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(v)) => { - Self::day_timestamp_nano(*v)? + Self::day_timestamp_nano(*v) } _ => { return Err(Error::new( @@ -1247,6 +1208,46 @@ mod test { Ok(()) } + #[test] + fn test_projection_timestamp_types_day_negative() -> Result<()> { + // 1969-12-30T23:59:59.5 + let micros = -86_400_500_000; + for (field_type, value) in [ + (Timestamp, Datum::timestamp_micros(micros)), + (Timestamptz, Datum::timestamptz_micros(micros)), + (TimestampNs, Datum::timestamp_nanos(micros * 1_000)), + (TimestamptzNs, Datum::timestamptz_nanos(micros * 1_000)), + ] { + let fixture = TestProjectionFixture::new( + Transform::Day, + "name", + NestedField::required(1, "value", Primitive(field_type)), + ); + + fixture.assert_projection( + &fixture.binary_predicate(PredicateOperator::LessThan, value.clone()), + Some("name <= 1969-12-31"), + )?; + + fixture.assert_projection( + &fixture.binary_predicate(PredicateOperator::LessThanOrEq, value.clone()), + Some("name <= 1969-12-31"), + )?; + + fixture.assert_projection( + &fixture.binary_predicate(PredicateOperator::Eq, value.clone()), + Some("name IN (1969-12-31, 1969-12-30)"), + )?; + + fixture.assert_projection( + &fixture.set_predicate(PredicateOperator::In, vec![value]), + Some("name IN (1969-12-31, 1969-12-30)"), + )?; + } + + Ok(()) + } + #[test] fn test_projection_timestamp_day_upper_bound() -> Result<()> { // 17501 @@ -2650,6 +2651,48 @@ mod test { test_timestamp_ns_and_tz_transform("2017-12-01T10:30:42.123000", &day, Datum::date(17501)); } + #[test] + fn test_transform_days_pre_epoch() { + let day = Box::new(super::Day) as BoxedTransformFunction; + let expected = [-1, -2, -2, -2, -2, -366, -365]; + + let micros = vec![ + -500_000, // 1969-12-31T23:59:59.500000 + -86_400_000_001, // 1969-12-30T23:59:59.999999 + -86_400_000_002, // 1969-12-30T23:59:59.999998 + -86_400_500_000, // 1969-12-30T23:59:59.500000 + -86_401_000_000, // 1969-12-30T23:59:59.000000 + -31_536_000_500_000, // 1968-12-31T23:59:59.500000 + -31_535_999_000_001, // 1969-01-01T00:00:00.999999, Iceberg Java gives -366 + ]; + let res = day + .transform(Arc::new(TimestampMicrosecondArray::from(micros.clone()))) + .unwrap(); + let res = res.as_any().downcast_ref::().unwrap(); + assert_eq!(res.values(), &expected); + for (v, d) in micros.into_iter().zip(expected) { + test_timestamp_and_tz_transform_using_i64(v, &day, Datum::date(d)); + } + + let nanos = vec![ + -500_000_000, // 1969-12-31T23:59:59.500000000 + -86_400_000_000_001, // 1969-12-30T23:59:59.999999999 + -86_400_000_000_002, // 1969-12-30T23:59:59.999999998 + -86_400_500_000_000, // 1969-12-30T23:59:59.500000000 + -86_401_000_000_000, // 1969-12-30T23:59:59.000000000 + -31_536_000_500_000_000, // 1968-12-31T23:59:59.500000000 + -31_535_999_000_000_001, // 1969-01-01T00:00:00.999999999, Iceberg Java gives -366 + ]; + let res = day + .transform(Arc::new(TimestampNanosecondArray::from(nanos.clone()))) + .unwrap(); + let res = res.as_any().downcast_ref::().unwrap(); + assert_eq!(res.values(), &expected); + for (v, d) in nanos.into_iter().zip(expected) { + test_timestamp_ns_and_tz_transform_using_i64(v, &day, Datum::date(d)); + } + } + #[test] fn test_transform_hours() { let hour = super::Hour; From f2409e9f5eaff1c643985013d927e17fdf16e6ec Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 6 Oct 2026 17:48:48 +0000 Subject: [PATCH 32/63] fix: fill schema-added nested columns with typed NULL arrays on read (#2635) * fix: fill schema-added nested columns with typed NULL arrays on read When a column of a nested type (list, map, or a struct with nested children) is added to the table schema after data files were written, reading those older files failed with "unexpected target column type": the helpers that materialize missing columns only handled primitive types plus structs with primitive-only children, via hand-written per-type NULL branches. Build the all-NULL column with arrow's new_null_array instead, which supports every Arrow type (including arbitrarily nested ones), and drop the per-type NULL branches from create_primitive_array_repeated and create_primitive_array_single_element. Closes #2618 * test(arrow): restructure #2618 nested-null test; trim comments Address review feedback on #2635: - Extract the evolved schema into `schema_with_added_nested_columns` and reorder the test to read top-to-bottom as a story (old file -> evolved schema -> expect typed NULLs), per @kevinjqliu. - Assert the all-NULL struct still carries its full nested shape (child fields `a` and the nested list `ys`), locking in that the type-preserving new_null_array fill keeps nested children. - Trim the over-explanatory comments on the new_null_array NULL-fill paths and note that the literal `1` is new_null_array's row count. * test: cover typed null helpers and clarify struct default support * test: compare full evolved schema and link default follow-up --- .../src/arrow/record_batch_transformer.rs | 207 +++++++++++++++++- crates/iceberg/src/arrow/value.rs | 201 ++++++----------- 2 files changed, 269 insertions(+), 139 deletions(-) diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index 19976a81c5..fe17d336ca 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -864,13 +864,24 @@ impl RecordBatchTransformer { return Err(invalid_data!("Missing required field: {}", iceberg_field.name)); } - let default_value = iceberg_field.initial_default.as_ref().and_then(|lit| { - if let Literal::Primitive(prim) = lit { - Some(prim.clone()) - } else { - None + // TODO: Support the spec-defined non-null struct initial default `{}` by + // creating a non-null struct and applying each child's initial default. + // Tracked in https://github.com/apache/iceberg-rust/issues/3261. + let default_value = match iceberg_field.initial_default.as_ref() { + None => None, + Some(Literal::Primitive(prim)) => Some(prim.clone()), + Some(_) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Cannot read field {} that is absent from the data file: \ + applying a non-primitive initial-default for {} is not yet \ + supported", + iceberg_field.name, iceberg_field.field_type + ), + )); } - }); + }; ColumnSource::Add { value: default_value, @@ -1143,12 +1154,14 @@ mod test { use arrow_cast::cast; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; - use super::field_with_id; + use super::{field_with_id, schema_to_arrow_schema}; use crate::arrow::build_partition_constant; use crate::arrow::record_batch_transformer::{ RecordBatchTransformer, RecordBatchTransformerBuilder, }; - use crate::spec::{Literal, NestedField, PrimitiveType, Schema, Struct, Type}; + use crate::spec::{ + ListType, Literal, MapType, NestedField, PrimitiveType, Schema, Struct, Type, + }; /// Helper to extract string values from either StringArray or RunEndEncoded /// Returns empty string for null values @@ -1360,6 +1373,63 @@ mod test { ); } + #[test] + fn schema_evolution_absent_struct_with_initial_default_errors() { + for required in [false, true] { + let added_field = NestedField::new( + 2, + "added_struct", + Type::Struct(crate::spec::StructType::new(vec![ + NestedField::optional(3, "child", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(42)) + .into(), + ])), + required, + ) + .with_initial_default(Literal::Struct(Struct::from_iter(vec![None]))); + let snapshot_schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + added_field.into(), + ]) + .build() + .unwrap(), + ); + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &[1, 2]).build(); + let file_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "id", + DataType::Int32, + false, + 1, + )])); + let file_batch = + RecordBatch::try_new(file_schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]) + .unwrap(); + + let err = transformer + .process_record_batch(file_batch) + .expect_err(&format!( + "required={required}: complex default should be rejected" + )); + assert_eq!( + err.kind(), + crate::ErrorKind::FeatureUnsupported, + "required={required}" + ); + assert!( + err.to_string().contains("Cannot read field added_struct"), + "required={required}: {err}" + ); + assert!( + err.to_string() + .contains("applying a non-primitive initial-default"), + "required={required}: {err}" + ); + } + } + #[test] fn schema_evolution_adds_struct_column_with_nulls() { // Test that when a struct column is added after data files are written, @@ -1437,6 +1507,127 @@ mod test { assert!(struct_column.is_null(2)); } + /// Evolved table schema for the #2618 regression test: `id` plus three + /// later-added optional nested columns — a list, a map, and a struct that + /// itself contains a nested list (`ys`). The nested-in-struct list is the + /// case a per-type NULL-fill would miss. + fn schema_with_added_nested_columns() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 2, + "xs", + Type::List(ListType { + element_field: NestedField::list_element( + 3, + Type::Primitive(PrimitiveType::Int), + false, + ) + .into(), + }), + ) + .into(), + NestedField::optional( + 4, + "props", + Type::Map(MapType { + key_field: NestedField::map_key_element( + 5, + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::map_value_element( + 6, + Type::Primitive(PrimitiveType::Int), + false, + ) + .into(), + }), + ) + .into(), + NestedField::optional( + 7, + "s", + Type::Struct(crate::spec::StructType::new(vec![ + NestedField::optional(8, "a", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::optional( + 9, + "ys", + Type::List(ListType { + element_field: NestedField::list_element( + 10, + Type::Primitive(PrimitiveType::Long), + false, + ) + .into(), + }), + ) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap() + } + + #[test] + fn schema_evolution_adds_list_map_and_nested_struct_columns_with_nulls() { + // Regression test for https://github.com/apache/iceberg-rust/issues/2618. + // + // The story the test tells, in order: + // 1. An old data file was written with only the `id` column. + // 2. The table schema has since evolved, adding optional list / map / + // struct columns (see `schema_with_added_nested_columns`). + // 3. Reading the old file against the evolved schema must fill those + // absent columns with typed all-NULL arrays — previously this errored + // with "unexpected target column type" for the nested types. + + // (1) The old data file: just `id`. + let file_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "id", + DataType::Int32, + false, + 1, + )])); + let file_batch = + RecordBatch::try_new(file_schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]) + .unwrap(); + + // (2) Read it against the evolved schema, projecting id + the three added columns. + let snapshot_schema = Arc::new(schema_with_added_nested_columns()); + let projected_iceberg_field_ids = [1, 2, 4, 7]; + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_iceberg_field_ids) + .build(); + let result = transformer.process_record_batch(file_batch).unwrap(); + + // (3a) `id` survives unchanged. + assert_eq!(result.num_columns(), 4); + assert_eq!(result.num_rows(), 3); + let id_column = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(id_column.values(), &[1, 2, 3]); + + // (3b) The added columns carry the evolved schema's Arrow types, including nested + // field names, nullability, and field IDs, and are all-NULL. + let expected_schema = schema_to_arrow_schema(&schema_with_added_nested_columns()).unwrap(); + assert_eq!(result.schema().as_ref(), &expected_schema); + for (idx, name) in [(1, "xs"), (2, "props"), (3, "s")] { + assert_eq!( + result.column(idx).null_count(), + 3, + "added nested column `{name}` should be all-NULL" + ); + } + } + pub fn source_record_batch() -> RecordBatch { RecordBatch::try_new( arrow_schema_promotion_addition_and_renaming_required(), diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index 080d1b99a6..7cbebf68c6 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -23,7 +23,7 @@ use arrow_array::{ LargeListArray, LargeStringArray, ListArray, MapArray, StringArray, StructArray, Time64MicrosecondArray, TimestampMicrosecondArray, TimestampNanosecondArray, new_null_array, }; -use arrow_buffer::{BooleanBuffer, NullBuffer}; +use arrow_buffer::BooleanBuffer; use arrow_schema::{DataType, FieldRef, TimeUnit}; use uuid::Uuid; @@ -526,24 +526,23 @@ pub(crate) fn create_primitive_array_single_element( data_type: &DataType, prim_lit: Option<&PrimitiveLiteral>, ) -> Result { + // No value: a single NULL of any (possibly nested) type. + if prim_lit.is_none() { + return Ok(new_null_array(data_type, 1)); + } match (data_type, prim_lit) { - (DataType::Null, None) => Ok(Arc::new(arrow_array::NullArray::new(1))), (DataType::Boolean, Some(PrimitiveLiteral::Boolean(v))) => { Ok(Arc::new(BooleanArray::from(vec![*v]))) } - (DataType::Boolean, None) => Ok(Arc::new(BooleanArray::from(vec![Option::::None]))), (DataType::Int32, Some(PrimitiveLiteral::Int(v))) => { Ok(Arc::new(Int32Array::from(vec![*v]))) } - (DataType::Int32, None) => Ok(Arc::new(Int32Array::from(vec![Option::::None]))), (DataType::Date32, Some(PrimitiveLiteral::Int(v))) => { Ok(Arc::new(Date32Array::from(vec![*v]))) } - (DataType::Date32, None) => Ok(Arc::new(Date32Array::from(vec![Option::::None]))), (DataType::Int64, Some(PrimitiveLiteral::Long(v))) => { Ok(Arc::new(Int64Array::from(vec![*v]))) } - (DataType::Int64, None) => Ok(Arc::new(Int64Array::from(vec![Option::::None]))), (DataType::Timestamp(TimeUnit::Microsecond, timezone), Some(PrimitiveLiteral::Long(v))) => { let array = TimestampMicrosecondArray::from(vec![*v]); if let Some(timezone) = timezone { @@ -552,14 +551,6 @@ pub(crate) fn create_primitive_array_single_element( Ok(Arc::new(array)) } } - (DataType::Timestamp(TimeUnit::Microsecond, timezone), None) => { - let array = TimestampMicrosecondArray::from(vec![Option::::None]); - if let Some(timezone) = timezone { - Ok(Arc::new(array.with_timezone(timezone.clone()))) - } else { - Ok(Arc::new(array)) - } - } (DataType::Timestamp(TimeUnit::Nanosecond, timezone), Some(PrimitiveLiteral::Long(v))) => { let array = TimestampNanosecondArray::from(vec![*v]); if let Some(timezone) = timezone { @@ -568,32 +559,18 @@ pub(crate) fn create_primitive_array_single_element( Ok(Arc::new(array)) } } - (DataType::Timestamp(TimeUnit::Nanosecond, timezone), None) => { - let array = TimestampNanosecondArray::from(vec![Option::::None]); - if let Some(timezone) = timezone { - Ok(Arc::new(array.with_timezone(timezone.clone()))) - } else { - Ok(Arc::new(array)) - } - } (DataType::Float32, Some(PrimitiveLiteral::Float(v))) => { Ok(Arc::new(Float32Array::from(vec![v.0]))) } - (DataType::Float32, None) => Ok(Arc::new(Float32Array::from(vec![Option::::None]))), (DataType::Float64, Some(PrimitiveLiteral::Double(v))) => { Ok(Arc::new(Float64Array::from(vec![v.0]))) } - (DataType::Float64, None) => Ok(Arc::new(Float64Array::from(vec![Option::::None]))), (DataType::Utf8, Some(PrimitiveLiteral::String(v))) => { Ok(Arc::new(StringArray::from(vec![v.as_str()]))) } - (DataType::Utf8, None) => Ok(Arc::new(StringArray::from(vec![Option::<&str>::None]))), (DataType::Binary, Some(PrimitiveLiteral::Binary(v))) => { Ok(Arc::new(BinaryArray::from_vec(vec![v.as_slice()]))) } - (DataType::Binary, None) => Ok(Arc::new(BinaryArray::from_opt_vec(vec![ - Option::<&[u8]>::None, - ]))), (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::Int128(v))) => { let array = Decimal128Array::from(vec![{ *v }]) .with_precision_and_scale(*precision, *scale) @@ -614,78 +591,6 @@ pub(crate) fn create_primitive_array_single_element( })?; Ok(Arc::new(array)) } - (DataType::Decimal128(precision, scale), None) => { - let array = Decimal128Array::from(vec![Option::::None]) - .with_precision_and_scale(*precision, *scale) - .map_err(|e| { - invalid_data!( - "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" - ) - })?; - Ok(Arc::new(array)) - } - (DataType::Struct(fields), None) => { - // Create a single-element StructArray with nulls - let null_arrays: Vec = fields - .iter() - .map(|f| { - // Recursively create null arrays for struct fields - // For primitive fields in structs, use simple null arrays (not REE within struct) - match f.data_type() { - DataType::Boolean => { - Ok(Arc::new(BooleanArray::from(vec![Option::::None])) - as ArrayRef) - } - DataType::Int32 | DataType::Date32 => { - Ok(Arc::new(Int32Array::from(vec![Option::::None])) as ArrayRef) - } - DataType::Int64 => { - Ok(Arc::new(Int64Array::from(vec![Option::::None])) as ArrayRef) - } - DataType::Timestamp(TimeUnit::Microsecond, timezone) => { - let array = TimestampMicrosecondArray::from(vec![Option::::None]); - if let Some(timezone) = timezone { - Ok(Arc::new(array.with_timezone(timezone.clone())) as ArrayRef) - } else { - Ok(Arc::new(array) as ArrayRef) - } - } - DataType::Timestamp(TimeUnit::Nanosecond, timezone) => { - let array = TimestampNanosecondArray::from(vec![Option::::None]); - if let Some(timezone) = timezone { - Ok(Arc::new(array.with_timezone(timezone.clone())) as ArrayRef) - } else { - Ok(Arc::new(array) as ArrayRef) - } - } - DataType::Float32 => { - Ok(Arc::new(Float32Array::from(vec![Option::::None])) as ArrayRef) - } - DataType::Float64 => { - Ok(Arc::new(Float64Array::from(vec![Option::::None])) as ArrayRef) - } - DataType::Utf8 => { - Ok(Arc::new(StringArray::from(vec![Option::<&str>::None])) as ArrayRef) - } - DataType::Binary => { - Ok( - Arc::new(BinaryArray::from_opt_vec(vec![Option::<&[u8]>::None])) - as ArrayRef, - ) - } - _ => Err(Error::new( - ErrorKind::Unexpected, - format!("Unsupported struct field type: {:?}", f.data_type()), - )), - } - }) - .collect::>>()?; - Ok(Arc::new(StructArray::new( - fields.clone(), - null_arrays, - Some(NullBuffer::new_null(1)), - ))) - } _ => Err(Error::new( ErrorKind::Unexpected, format!("Unsupported constant type combination: {data_type:?} with {prim_lit:?}"), @@ -702,6 +607,10 @@ pub(crate) fn create_primitive_array_repeated( prim_lit: Option<&PrimitiveLiteral>, num_rows: usize, ) -> Result { + // No value to repeat: an all-NULL column of any (possibly nested) type. + if prim_lit.is_none() { + return Ok(new_null_array(data_type, num_rows)); + } Ok(match (data_type, prim_lit) { // --- Primitive Some arms --- (DataType::Boolean, Some(PrimitiveLiteral::Boolean(value))) => { @@ -805,37 +714,6 @@ pub(crate) fn create_primitive_array_repeated( ) } - // --- Special-case None arms --- - (DataType::Decimal128(precision, scale), None) => { - let vals: Vec> = vec![None; num_rows]; - Arc::new( - Decimal128Array::from(vals) - .with_precision_and_scale(*precision, *scale) - .map_err(|e| { - invalid_data!( - "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" - ) - })?, - ) - } - (DataType::Struct(fields), None) => { - // Create a StructArray filled with nulls, recursively creating null children - let null_arrays: Vec = fields - .iter() - .map(|field| create_primitive_array_repeated(field.data_type(), None, num_rows)) - .collect::>>()?; - - Arc::new(StructArray::new( - fields.clone(), - null_arrays, - Some(NullBuffer::new_null(num_rows)), - )) - } - (DataType::Null, None) => Arc::new(arrow_array::NullArray::new(num_rows)), - - // --- Catch-all null arm: use arrow-rs new_null_array for any remaining DataType --- - (dt, None) => new_null_array(dt, num_rows), - (dt, _) => { return Err(Error::new( ErrorKind::Unexpected, @@ -1754,6 +1632,67 @@ mod test { } } + #[test] + fn test_create_null_arrays_preserve_type_and_length() { + let data_types = [ + DataType::Decimal128(10, 2), + DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), + DataType::Struct( + vec![ + Field::new("a", DataType::Utf8, true), + Field::new( + "ys", + DataType::List(Arc::new(Field::new("element", DataType::Int64, true))), + true, + ), + ] + .into(), + ), + DataType::Null, + ]; + + // NullArray has no validity bitmap; logical_null_count includes its implicit NULLs. + for data_type in data_types { + let single = create_primitive_array_single_element(&data_type, None) + .unwrap_or_else(|err| panic!("single, type={data_type:?}: {err}")); + assert_eq!(single.data_type(), &data_type, "single, type={data_type:?}"); + assert_eq!(single.len(), 1, "single, type={data_type:?}"); + if data_type != DataType::Null { + assert_eq!(single.null_count(), 1, "single, type={data_type:?}"); + } + assert_eq!(single.logical_null_count(), 1, "single, type={data_type:?}"); + + for num_rows in [0, 1, 3] { + let repeated = create_primitive_array_repeated(&data_type, None, num_rows) + .unwrap_or_else(|err| { + panic!("repeated, type={data_type:?}, rows={num_rows}: {err}") + }); + assert_eq!( + repeated.data_type(), + &data_type, + "repeated, type={data_type:?}, rows={num_rows}" + ); + assert_eq!( + repeated.len(), + num_rows, + "repeated, type={data_type:?}, rows={num_rows}" + ); + if data_type != DataType::Null { + assert_eq!( + repeated.null_count(), + num_rows, + "repeated, type={data_type:?}, rows={num_rows}" + ); + } + assert_eq!( + repeated.logical_null_count(), + num_rows, + "repeated, type={data_type:?}, rows={num_rows}" + ); + } + } + } + #[test] fn test_create_null_array_rejects_non_null_literal() { let literal = Some(PrimitiveLiteral::Int(1)); From 78fafa99fa77903545dfd3301c8db716f3482b2a Mon Sep 17 00:00:00 2001 From: Yuhao Su <31772373+yuhao-su@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:11:47 +0000 Subject: [PATCH 33/63] fix(storage-azdls): drop the leading slash from relative paths (#3311) ADLS relative paths kept the leading `/` of the URL path, unlike the other backends. OpenDAL's Deleter passes paths to the service without normalizing them, so FileIO::delete_stream sent `DELETE //dir/file`, which ADLS rejects with 400 InvalidUri. Co-authored-by: Claude Opus 5.5 --- crates/storage/opendal/src/azdls.rs | 28 +++++++++++++++++++++++----- crates/storage/opendal/src/lib.rs | 5 ++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/crates/storage/opendal/src/azdls.rs b/crates/storage/opendal/src/azdls.rs index 6d39320df8..88a197fea8 100644 --- a/crates/storage/opendal/src/azdls.rs +++ b/crates/storage/opendal/src/azdls.rs @@ -101,8 +101,7 @@ pub(crate) fn azdls_create_operator<'a>( // including their filesystem and account name. // OpenDAL's operator methods expect only the relative path, so we split it // off and save it for later use. - let relative_path_len = path.path.len(); - let (_, relative_path) = absolute_path.split_at(absolute_path.len() - relative_path_len); + let relative_path = path.relative_path(absolute_path); Ok((op, relative_path)) } @@ -237,6 +236,13 @@ impl AzureStoragePath { self.endpoint_suffix ) } + + /// The part of `absolute_path` below the filesystem, without a leading `/`, + /// like the relative paths of the other backends. + pub(crate) fn relative_path<'a>(&self, absolute_path: &'a str) -> &'a str { + let path = &absolute_path[absolute_path.len() - self.path.len()..]; + path.strip_prefix('/').unwrap_or(path) + } } impl FromStr for AzureStoragePath { @@ -395,7 +401,19 @@ mod tests { ..Default::default() }, ), - Some(("myfs", "/path/to/file.parquet")), + Some(("myfs", "path/to/file.parquet")), + ), + ( + "filesystem root", + ( + "abfss://myfs@myaccount.dfs.core.windows.net/", + AzdlsConfig { + account_name: Some("myaccount".to_string()), + endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), + ..Default::default() + }, + ), + Some(("myfs", "")), ), ( "different account", @@ -445,7 +463,7 @@ mod tests { ..Default::default() }, ), - Some(("myfs", "/path/to/file.parquet")), + Some(("myfs", "path/to/file.parquet")), ), ( "scheme differs from a previously-configured one is accepted", @@ -459,7 +477,7 @@ mod tests { ..Default::default() }, ), - Some(("myfs", "/path/to/file.parquet")), + Some(("myfs", "path/to/file.parquet")), ), ]; diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index f58d6f534c..9e01552016 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -603,8 +603,7 @@ impl OpenDalStorage { OpenDalStorage::Azdls { config, .. } => { let azure_path = path.parse::()?; match_path_with_config(&azure_path, config)?; - let relative_path_len = azure_path.path.len(); - Ok(&path[path.len() - relative_path_len..]) + Ok(azure_path.relative_path(path)) } #[cfg(feature = "opendal-hf")] OpenDalStorage::Hf { .. } => { @@ -1176,7 +1175,7 @@ mod tests { storage .relativize_path("abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet") .unwrap(), - "/path/to/file.parquet" + "path/to/file.parquet" ); } } From 05e03d179521260a1f4a230cd35542c55b009dc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:16:36 +0000 Subject: [PATCH 34/63] chore(deps): Bump reqwest from 0.12.28 to 0.13.4 (#3321) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.28 to 0.13.4. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.12.28...v0.13.4) --- updated-dependencies: - dependency-name: reqwest dependency-version: 0.13.4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b2115a713..e2900acb2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1407,7 +1407,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -2662,7 +2662,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -4640,7 +4640,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -4679,7 +4679,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] @@ -7069,7 +7069,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] From 08cc6986bb3e3aef61a24fc38f869d124c1c0c05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:17:04 +0000 Subject: [PATCH 35/63] chore(deps): Bump crate-ci/typos from 1.50.1 to 1.50.2 (#3320) Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.50.1 to 1.50.2. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/main/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/d43b6c087ac471e2ea7b8af622ff15f05c0c365b...512fc24f32f44ab01972217aaaf3dc86ec234d53) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.50.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c181efd6d..72e6a070e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: run: cargo machete - name: Check typos - uses: crate-ci/typos@d43b6c087ac471e2ea7b8af622ff15f05c0c365b # v1.50.1 + uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2 clippy: needs: lint From 73cadf8dda9a4b8fa3f2cb941272dbf64206402a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:17:24 +0000 Subject: [PATCH 36/63] chore(deps): Bump zizmorcore/zizmor-action from 0.6.3 to 0.6.4 (#3319) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.6.3 to 0.6.4. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/70fb788f84895a7701f5643d103d587e460b5c99...cc914d7f3750a2d13d75c7f184a1060aa0e9d482) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 5574a9f95a..5558526351 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -37,6 +37,6 @@ jobs: persist-credentials: false - name: Run zizmor 🌈 - uses: zizmorcore/zizmor-action@70fb788f84895a7701f5643d103d587e460b5c99 # v0.6.3 + uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 with: advanced-security: false From 42f8d81f5038d390cb92180e4bea629ea766750d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:17:47 +0000 Subject: [PATCH 37/63] chore(deps): Bump astral-sh/setup-uv from 10.0.1 to 10.2.0 (#3318) Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 10.0.1 to 10.2.0. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/20cfd1bf945f4377ade1205e4dbc17946fc9a30d...c18668ad3cf93ea998bef934396af7bb5c839dc7) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 10.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/bindings_python_ci.yml | 4 ++-- .github/workflows/release_python.yml | 2 +- .github/workflows/release_python_nightly.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 68586abff1..e0c5fab629 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -51,7 +51,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: "0.9.3" enable-cache: true @@ -98,7 +98,7 @@ jobs: working-directory: "bindings/python" command: build args: --out dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: "0.9.3" enable-cache: true diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 758bdff21b..b89048bfbd 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -190,7 +190,7 @@ jobs: working-directory: "bindings/python" command: build args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: "0.9.3" enable-cache: true diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index ab2b2ff9c1..b33d64ddff 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -124,7 +124,7 @@ jobs: command: build args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: "0.9.3" enable-cache: true From 19f705726ab483b6285014d5cb878c541de72079 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:18:09 +0000 Subject: [PATCH 38/63] chore(deps): Bump taiki-e/install-action from 2.87.8 to 2.87.21 (#3317) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.87.8 to 2.87.21. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/d438492cf8a250514fa2d34b30bc3c0dc37c65ff...4cef1412cce204788f482e778a0b9187f9626a29) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.87.20 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72e6a070e6..2c57b4799c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: - name: Install lint tools # Keep versions in sync with the local install targets in Makefile. - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 with: tool: taplo@0.9.3,cargo-machete@0.7.0,cargo-deny@0.19.9 @@ -238,7 +238,7 @@ jobs: - name: Install cargo-nextest if: matrix.test-suite.name == 'default' - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 with: tool: cargo-nextest From 4ee006f50c372d8311e70f6afc67bf2499372b83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:18:35 +0000 Subject: [PATCH 39/63] chore(deps): Bump the codeql-action group with 2 updates (#3316) Bumps the codeql-action group with 2 updates: [github/codeql-action/init](https://github.com/github/codeql-action) and [github/codeql-action/analyze](https://github.com/github/codeql-action). Updates `github/codeql-action/init` from 4.37.9 to 4.38.2 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/cdf488f595d80d6e07e03d4674febd5ab45fa938...2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2) Updates `github/codeql-action/analyze` from 4.37.9 to 4.38.2 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/cdf488f595d80d6e07e03d4674febd5ab45fa938...2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2) --- updated-dependencies: - dependency-name: github/codeql-action/init dependency-version: 4.38.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: codeql-action - dependency-name: github/codeql-action/analyze dependency-version: 4.38.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: codeql-action ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index efd720e534..9f512b57cd 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -46,11 +46,11 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 with: category: "/language:actions" From ab047b5e2f6e2ba6b999731653a3f7722b84e6f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:19:11 +0000 Subject: [PATCH 40/63] chore(deps): Bump urllib3 from 2.7.0 to 2.8.0 in /bindings/python (#3313) Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.7.0 to 2.8.0. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.7.0...2.8.0) --- updated-dependencies: - dependency-name: urllib3 dependency-version: 2.8.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bindings/python/uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock index 31569dc790..1a532bbda7 100644 --- a/bindings/python/uv.lock +++ b/bindings/python/uv.lock @@ -952,11 +952,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] From 776648bee4c52ece8a057cfcf0d7756416299ba8 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 8 Oct 2026 11:09:32 +0000 Subject: [PATCH 41/63] docs: add agent guidance to reduce verbosity (#3337) * docs: add agent guidance on code comments and API surface Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs: add agent guidance on PR descriptions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs: separate always-flag review rules Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs: soften always-flag wording Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs: drop refactor test from comment guidance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs: base pub guidance on intended API Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs: clarify visibility guidance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs: add reuse and test duplication guidance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs: trim reuse guidance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Apply batched suggestions from code review Co-authored-by: Matt Butrovich --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: blackmwk <257669749+blackmwk@users.noreply.github.com> Co-authored-by: Matt Butrovich --- .github/copilot-instructions.md | 12 +++++++++++ AGENTS.md | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 3744d6f435..c00ef406b2 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -19,3 +19,15 @@ When reviewing pull requests, follow these rules to avoid noise and redundancy: - First-time contributors: For users new to this repository, explicitly instruct them to "Please check and address all review comments in this PR." + +Also flag these, once per PR with the locations: + +- Comment churn: Code comments or docs that restate the code, narrate + history, or justify review decisions. +- Unrelated edits: Renames, reformatting, or reordering of code the + change does not otherwise need. +- Duplication: New helpers, types, or tests that duplicate existing ones. +- API growth: New public API the change does not need. Check new entries + in `public-api.txt`. +- PR description: Descriptions that restate the diff or pad the + template. Mention it in the review summary. diff --git a/AGENTS.md b/AGENTS.md index 22ddb2bbd6..5a1106e0b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,44 @@ This file provides repository-specific guidance for automated agents working in this repository. +## Comments, docs, and diff scope + +Default to no comment. Leave existing comments as they are unless you change +the code they describe or they are wrong. + +- Comment only what the code cannot say: invariants, non-obvious + constraints, links to external behavior. Do not restate the code. +- Do not describe how the code changed or explain review history + ("previously", "instead of", "unlike X"). Put that in the commit message + or PR description. +- Public items get a one-line doc summary of what they do for the caller, + not how. Expand only for contract the caller must know, such as the + errors it returns (`# Errors`) or when it panics (`# Panics`). +- Test names say what they check. Do not narrate asserts. +- Cover the edge cases and error paths the change adds, not only the + happy path. +- Build the expected value and assert equality with it, rather than + checking fields one at a time. +- Do not add tests that duplicate existing coverage. +- Do not rename, reformat, refactor, or reorder code unrelated to the + change. A bug fix does not carry an API redesign. +- Reuse existing helpers, types, and test fixtures (such as + `TableTestFixture`) before adding new ones. Use arrow-rs kernels + instead of reimplementing them. +- Use the narrowest visibility that works. Make items `pub` only when they + are part of the API the change intends to expose. Run + `make generate-public-api` to update each crate's `public-api.txt`, and + check its diff for API growth. + +## Pull requests + +- Follow the PR template. Keep each section short: the problem, the + approach, and anything a reviewer would not guess from the diff. In + AI Disclosure, name the areas of uncertainty or assumptions a reviewer + should focus on. +- Do not restate the diff: no file-by-file walkthroughs, per-function + summaries, or "changes made" checklists. + ## Security Model When assessing potential vulnerabilities or calibrating automated security From 7f9691fc7ed395ecb062a14ae918cb3707f22d30 Mon Sep 17 00:00:00 2001 From: Jannik Steinmann Date: Thu, 8 Oct 2026 23:55:44 +0000 Subject: [PATCH 42/63] feat(rest_catalog::auth) Contextual Sessions (#3081) * Add trait method `AuthManager::contextual_session` * Implement contextual session handling * Produce an auth_session per operation and re-use it for multiple Http requests * Rename client to RestCatalogClient * Mark context in contextual_session as used in API docs * Drop Java-derived comment on releasing resources * Revert "Produce an auth_session per operation and re-use it for multiple Http" This reverts commit 32a9dc5f2f1e35012bdcc48fcd9167af0fcc585b. * Expand test coverage * Pass auth session to client on each call This prevents cloning the HeaderMap of the client because a new session is passed for every HTTP request anyway * Leave post_form unauthenticated * Implementor guidance --- crates/catalog/rest/public-api.txt | 27 +- crates/catalog/rest/src/auth/mod.rs | 39 ++- crates/catalog/rest/src/catalog.rs | 404 +++++++++++++++++++++++----- crates/catalog/rest/src/client.rs | 120 +++------ 4 files changed, 422 insertions(+), 168 deletions(-) diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index 8afd581bcd..47393b5738 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -127,8 +127,6 @@ pub fn iceberg_catalog_rest::ErrorResponse::deserialize<__D>(__deserializer: __D pub struct iceberg_catalog_rest::HttpClient impl iceberg_catalog_rest::HttpClient pub async fn iceberg_catalog_rest::HttpClient::post_form(&self, url: &str, headers: &http::header::map::HeaderMap, form: &std::collections::hash::map::HashMap<&str, &str>) -> iceberg::error::Result -pub fn iceberg_catalog_rest::HttpClient::with_auth_session(&self, auth_session: alloc::sync::Arc) -> Self -pub fn iceberg_catalog_rest::HttpClient::without_auth_session(&self) -> Self impl core::clone::Clone for iceberg_catalog_rest::HttpClient pub fn iceberg_catalog_rest::HttpClient::clone(&self) -> iceberg_catalog_rest::HttpClient impl core::fmt::Debug for iceberg_catalog_rest::HttpClient @@ -222,6 +220,7 @@ impl core::fmt::Debug for iceberg_catalog_rest::NoopAuthManager pub fn iceberg_catalog_rest::NoopAuthManager::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::NoopAuthManager pub fn iceberg_catalog_rest::NoopAuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::NoopAuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::NoopAuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub struct iceberg_catalog_rest::OAuth2Manager impl iceberg_catalog_rest::OAuth2Manager @@ -234,6 +233,7 @@ impl core::fmt::Debug for iceberg_catalog_rest::OAuth2Manager pub fn iceberg_catalog_rest::OAuth2Manager::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::OAuth2Manager pub fn iceberg_catalog_rest::OAuth2Manager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::OAuth2Manager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::OAuth2Manager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub struct iceberg_catalog_rest::RegisterTableRequest pub iceberg_catalog_rest::RegisterTableRequest::metadata_location: alloc::string::String @@ -304,21 +304,21 @@ pub struct iceberg_catalog_rest::RestSessionCatalog impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalog pub fn iceberg_catalog_rest::RestSessionCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::catalog::session::SessionCatalog for iceberg_catalog_rest::RestSessionCatalog -pub fn iceberg_catalog_rest::RestSessionCatalog::create_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::create_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent, creation: iceberg::catalog::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::drop_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::create_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::create_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent, creation: iceberg::catalog::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::drop_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::RestSessionCatalog::drop_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::get_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::list_namespaces<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, parent: core::option::Option<&'life2 iceberg::catalog::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::list_tables<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::load_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, table_ident: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::get_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::list_namespaces<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, parent: core::option::Option<&'life2 iceberg::catalog::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::list_tables<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::load_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table_ident: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::RestSessionCatalog::namespace_exists<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, ns: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::RestSessionCatalog::purge_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::register_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, table_ident: &'life2 iceberg::catalog::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::rename_table<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, src: &'life2 iceberg::catalog::TableIdent, dest: &'life3 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::register_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table_ident: &'life2 iceberg::catalog::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::rename_table<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, src: &'life2 iceberg::catalog::TableIdent, dest: &'life3 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait pub fn iceberg_catalog_rest::RestSessionCatalog::table_exists<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::RestSessionCatalog::update_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, _namespace: &'life2 iceberg::catalog::NamespaceIdent, _properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_catalog_rest::RestSessionCatalog::update_table<'life0, 'life1, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::update_table<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub struct iceberg_catalog_rest::RestSessionCatalogBuilder impl iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send @@ -385,12 +385,15 @@ pub const iceberg_catalog_rest::REST_CATALOG_PROP_URI: &str pub const iceberg_catalog_rest::REST_CATALOG_PROP_WAREHOUSE: &str pub trait iceberg_catalog_rest::AuthManager: core::fmt::Debug + core::marker::Send + core::marker::Sync pub fn iceberg_catalog_rest::AuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::AuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::AuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::NoopAuthManager pub fn iceberg_catalog_rest::NoopAuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::NoopAuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::NoopAuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::OAuth2Manager pub fn iceberg_catalog_rest::OAuth2Manager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::OAuth2Manager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::OAuth2Manager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub trait iceberg_catalog_rest::AuthSession: core::fmt::Debug + core::marker::Send + core::marker::Sync pub fn iceberg_catalog_rest::AuthSession::authenticate<'life0, 'life1, 'async_trait>(&'life0 self, request: &'life1 mut iceberg_catalog_rest::HttpRequest) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs index b15dcdb480..b4c0cf0be2 100644 --- a/crates/catalog/rest/src/auth/mod.rs +++ b/crates/catalog/rest/src/auth/mod.rs @@ -25,7 +25,7 @@ use std::fmt::Debug; use std::sync::Arc; use async_trait::async_trait; -use iceberg::Result; +use iceberg::{Result, SessionContext}; pub use oauth2::OAuth2Manager; use crate::client::HttpClient; @@ -44,11 +44,13 @@ pub const AUTH_TYPE_OAUTH2: &str = "oauth2"; /// [`RestCatalogBuilder::with_auth_manager`](crate::RestCatalogBuilder::with_auth_manager) or /// [`RestSessionCatalogBuilder::with_auth_manager`](crate::RestSessionCatalogBuilder::with_auth_manager). /// Catalog initialization calls [`AuthManager::catalog_session`] exactly once; -/// later sessions may rely on the state established by that call. +/// the context-specific sessions that [`AuthManager::contextual_session`] +/// derives from it may rely on the state established by that call. /// -/// Both methods are handed the catalog's [`HttpClient`], which an -/// implementation may reuse for its own requests (e.g. a token exchange) so -/// that they share the catalog's connection pool and configuration. +/// [`Self::init_session`] and [`Self::catalog_session`] are handed the +/// catalog's [`HttpClient`], which an implementation may clone and store for +/// its own requests (e.g. a token exchange) so that they share the catalog's +/// connection pool and configuration. #[async_trait] pub trait AuthManager: Debug + Send + Sync { /// Session used for the initial `/v1/config` handshake, given the @@ -73,6 +75,33 @@ pub trait AuthManager: Debug + Send + Sync { client: &HttpClient, props: &HashMap, ) -> Result>; + + /// Returns the authentication session for a specific context. + /// + /// The catalog calls this method only after [`Self::catalog_session`] has + /// succeeded. `catalog_session` is the catalog session returned by this + /// manager. If the context does not require different authentication, + /// implementations should return `catalog_session` unchanged. + /// + /// The catalog does not cache the returned session. Implementations should + /// cache context-specific sessions internally using [`SessionContext::session_id`] + /// and are responsible for eviction. Reusing a session ID with different + /// context may therefore return the previously cached session. + /// + /// The catalog calls this method for every request without serializing + /// calls that share a session ID. Caching implementations must guard + /// against concurrently creating multiple sessions for the same context. + /// + /// No [`HttpClient`] is passed. Implementations that need one, e.g. for a + /// token exchange, should store the client handed to [`Self::catalog_session`]. + async fn contextual_session( + &self, + context: &SessionContext, + catalog_session: Arc, + ) -> Result> { + let _ = context; + Ok(catalog_session) + } } /// Authenticates outgoing REST catalog requests. diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index cc5973316a..dcc89823ff 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -39,7 +39,9 @@ use reqwest::{Client, Method, StatusCode, Url}; use tokio::sync::OnceCell; use typed_builder::TypedBuilder; -use crate::auth::{AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, AuthManager, NoopAuthManager, OAuth2Manager}; +use crate::auth::{ + AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, AuthManager, AuthSession, NoopAuthManager, OAuth2Manager, +}; use crate::client::{ HttpClient, deserialize_catalog_response, deserialize_unexpected_catalog_error, }; @@ -416,10 +418,14 @@ pub(crate) fn oauth_params_from_props(props: &HashMap) -> HashMa params } -#[derive(Debug)] -struct RestClient { - /// Carries the session the auth manager derived from the merged - /// configuration, so every request below is authenticated. +struct RestCatalogClient { + /// The manager that created `catalog_session`; retained so each request can + /// derive authentication for its [`SessionContext`]. + auth_manager: Arc, + /// The catalog-wide session passed to [`AuthManager::contextual_session`]. + catalog_session: Arc, + /// Shared HTTP transport and configuration. It remains unauthenticated; + /// request-time clones attach the contextual session. http_client: HttpClient, /// Runtime config is fetched from rest server and stored here. /// @@ -429,9 +435,21 @@ struct RestClient { endpoints: HashSet, } -impl RestClient { - /// Initializes the runtime config, advertised endpoints, and authentication - /// sessions shared by one REST catalog instance. +impl Debug for RestCatalogClient { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Auth managers and sessions may contain secrets, so keep them out of + // the catalog's derived Debug output just as HttpClient does. + f.debug_struct("RestCatalogClient") + .field("http_client", &self.http_client) + .field("config", &self.config) + .field("endpoints", &self.endpoints) + .finish_non_exhaustive() + } +} + +impl RestCatalogClient { + /// Initializes the runtime config, advertised endpoints, auth manager, and + /// catalog authentication session shared by one REST catalog instance. async fn init( user_config: &RestCatalogConfig, auth_manager: Arc, @@ -442,16 +460,9 @@ impl RestClient { // it before deriving the catalog session. let catalog_config = { let init_session = auth_manager - .init_session( - &http_client.without_auth_session(), - &Self::auth_props(user_config), - ) + .init_session(&http_client, &Self::auth_props(user_config)) .await?; - Self::load_config( - &http_client.with_auth_session(Arc::from(init_session)), - user_config, - ) - .await? + Self::load_config(&http_client, init_session.as_ref(), user_config).await? }; // Use the advertised endpoints as-is, falling back to // `DEFAULT_ENDPOINTS` when absent or empty. @@ -463,16 +474,15 @@ impl RestClient { let http_client = http_client.update_with(&config)?; // The manager is handed an unauthenticated client: its own // requests must not be signed by the session it is deriving. - let session = auth_manager - .catalog_session( - &http_client.without_auth_session(), - &Self::auth_props(&config), - ) + let catalog_session = auth_manager + .catalog_session(&http_client, &Self::auth_props(&config)) .await?; Ok(Self { + auth_manager, + catalog_session, config, - http_client: http_client.with_auth_session(session), + http_client, endpoints, }) } @@ -480,12 +490,22 @@ impl RestClient { /// Testing only: the bearer token the catalog session would attach. #[cfg(test)] async fn token(&self) -> Option { - self.http_client.token().await + self.http_client.token(self.catalog_session.as_ref()).await } - /// Sends `request`, authenticated by the client's session. - async fn query_catalog(&self, request: HttpRequest) -> Result { - self.http_client.query_catalog(request).await + /// Sends `request` with the authentication derived for `context`. + async fn query_catalog( + &self, + context: &SessionContext, + request: HttpRequest, + ) -> Result { + let session = self + .auth_manager + .contextual_session(context, Arc::clone(&self.catalog_session)) + .await?; + self.http_client + .query_catalog(session.as_ref(), request) + .await } /// The properties handed to the [`AuthManager`], with the catalog `uri` @@ -512,6 +532,7 @@ impl RestClient { /// It's required for a REST catalog to update its config after creation. async fn load_config( http_client: &HttpClient, + auth_session: &dyn AuthSession, user_config: &RestCatalogConfig, ) -> Result { let mut request_builder = http_client.request(Method::GET, user_config.config_endpoint()); @@ -522,7 +543,7 @@ impl RestClient { let request = HttpRequest::build(request_builder)?; - let http_response = http_client.query_catalog(request).await?; + let http_response = http_client.query_catalog(auth_session, request).await?; match http_response.status() { StatusCode::OK => deserialize_catalog_response(http_response), @@ -575,7 +596,7 @@ impl RestCatalog { } #[cfg(test)] - async fn client(&self) -> Result<&RestClient> { + async fn client(&self) -> Result<&RestCatalogClient> { self.inner.client().await } } @@ -687,18 +708,20 @@ impl Catalog for RestCatalog { /// REST catalog implementation of [`SessionCatalog`]. /// -/// Each operation accepts a [`SessionContext`]. REST configuration, authentication sessions, -/// and the HTTP client are initialized lazily once per catalog and shared across all operations. +/// Each operation accepts a [`SessionContext`]. REST configuration, the auth manager, the catalog +/// authentication session, and the HTTP client are initialized lazily once per catalog. Each +/// REST request uses the contextual authentication session returned by the manager. #[derive(Debug)] pub struct RestSessionCatalog { - /// Injected through [`RestSessionCatalogBuilder::with_auth_manager`]; otherwise - /// one is resolved from `rest.auth.type` when the client is built. - auth_manager: Option>, + /// Builder-supplied override retained so lazy client initialization can + /// clone it into the runtime state. When absent, a manager is resolved from + /// `rest.auth.type` during initialization. + auth_manager_override: Option>, /// User config is stored as-is and never changed. /// /// It could be different from the config fetched from the server and used at runtime. user_config: RestCatalogConfig, - client: OnceCell, + client: OnceCell, /// Storage factory for creating FileIO instances. storage_factory: Option>, runtime: Runtime, @@ -710,13 +733,13 @@ impl RestSessionCatalog { /// Creates a `RestSessionCatalog` from a [`RestCatalogConfig`]. fn new( config: RestCatalogConfig, - auth_manager: Option>, + auth_manager_override: Option>, storage_factory: Option>, runtime: Runtime, kms_client: Option>, ) -> Self { Self { - auth_manager: auth_manager.map(Arc::from), + auth_manager_override: auth_manager_override.map(Arc::from), user_config: config, client: OnceCell::new(), storage_factory, @@ -728,7 +751,7 @@ impl RestSessionCatalog { /// Sends a DELETE request for the given table, optionally requesting purge. async fn delete_table( &self, - _context: &SessionContext, + context: &SessionContext, table: &TableIdent, purge: bool, ) -> Result<()> { @@ -743,7 +766,7 @@ impl RestSessionCatalog { } let request = HttpRequest::build(request_builder)?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), @@ -783,7 +806,7 @@ impl RestSessionCatalog { /// Resolves the auth manager: a `with_auth_manager` override wins, /// otherwise one is built from the `rest.auth.type` configuration. fn resolve_auth_manager(&self) -> Result> { - if let Some(auth_manager) = &self.auth_manager { + if let Some(auth_manager) = &self.auth_manager_override { return Ok(auth_manager.clone()); } let config = &self.user_config; @@ -811,11 +834,11 @@ impl RestSessionCatalog { } } - /// Gets the [`RestClient`] from the catalog. - async fn client(&self) -> Result<&RestClient> { + /// Gets the [`RestCatalogClient`] from the catalog. + async fn client(&self) -> Result<&RestCatalogClient> { self.client .get_or_try_init(|| async { - RestClient::init(&self.user_config, self.resolve_auth_manager()?).await + RestCatalogClient::init(&self.user_config, self.resolve_auth_manager()?).await }) .await } @@ -829,9 +852,14 @@ impl RestSessionCatalog { /// Issue a `HEAD` request to `url` and interpret it as an existence check: /// `2xx` means it exists, `404` means it doesn't. - async fn check_exists_via_head(&self, client: &RestClient, url: String) -> Result { + async fn check_exists_via_head( + &self, + context: &SessionContext, + client: &RestCatalogClient, + url: String, + ) -> Result { let request = HttpRequest::build(client.http_client.request(Method::HEAD, url))?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(true), @@ -891,7 +919,7 @@ impl RestSessionCatalog { impl SessionCatalog for RestSessionCatalog { async fn list_namespaces( &self, - _context: &SessionContext, + context: &SessionContext, parent: Option<&NamespaceIdent>, ) -> Result> { let client = self.client().await?; @@ -911,7 +939,9 @@ impl SessionCatalog for RestSessionCatalog { request = request.query(&[("pageToken", token)]); } - let http_response = client.query_catalog(HttpRequest::build(request)?).await?; + let http_response = client + .query_catalog(context, HttpRequest::build(request)?) + .await?; match http_response.status() { StatusCode::OK => { @@ -945,7 +975,7 @@ impl SessionCatalog for RestSessionCatalog { async fn create_namespace( &self, - _context: &SessionContext, + context: &SessionContext, namespace: &NamespaceIdent, properties: HashMap, ) -> Result { @@ -961,7 +991,7 @@ impl SessionCatalog for RestSessionCatalog { }), )?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::OK => { @@ -981,7 +1011,7 @@ impl SessionCatalog for RestSessionCatalog { async fn get_namespace( &self, - _context: &SessionContext, + context: &SessionContext, namespace: &NamespaceIdent, ) -> Result { let client = self.client().await?; @@ -992,7 +1022,7 @@ impl SessionCatalog for RestSessionCatalog { .request(Method::GET, client.config.namespace_endpoint(namespace)), )?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::OK => { @@ -1028,7 +1058,7 @@ impl SessionCatalog for RestSessionCatalog { } let client = self.client().await?; - self.check_exists_via_head(client, client.config.namespace_endpoint(ns)) + self.check_exists_via_head(context, client, client.config.namespace_endpoint(ns)) .await } @@ -1046,7 +1076,7 @@ impl SessionCatalog for RestSessionCatalog { async fn drop_namespace( &self, - _context: &SessionContext, + context: &SessionContext, namespace: &NamespaceIdent, ) -> Result<()> { let client = self.client().await?; @@ -1057,7 +1087,7 @@ impl SessionCatalog for RestSessionCatalog { .request(Method::DELETE, client.config.namespace_endpoint(namespace)), )?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), @@ -1074,7 +1104,7 @@ impl SessionCatalog for RestSessionCatalog { async fn list_tables( &self, - _context: &SessionContext, + context: &SessionContext, namespace: &NamespaceIdent, ) -> Result> { let client = self.client().await?; @@ -1089,7 +1119,9 @@ impl SessionCatalog for RestSessionCatalog { request = request.query(&[("pageToken", token)]); } - let http_response = client.query_catalog(HttpRequest::build(request)?).await?; + let http_response = client + .query_catalog(context, HttpRequest::build(request)?) + .await?; match http_response.status() { StatusCode::OK => { @@ -1129,7 +1161,7 @@ impl SessionCatalog for RestSessionCatalog { /// the value provided locally to the `RestSessionCatalog` will take precedence. async fn create_table( &self, - _context: &SessionContext, + context: &SessionContext, namespace: &NamespaceIdent, creation: TableCreation, ) -> Result { @@ -1152,7 +1184,7 @@ impl SessionCatalog for RestSessionCatalog { }), )?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; let response = match http_response.status() { StatusCode::OK => deserialize_catalog_response::(http_response)?, @@ -1214,7 +1246,7 @@ impl SessionCatalog for RestSessionCatalog { /// value provided locally to the `RestSessionCatalog` will take precedence. async fn load_table( &self, - _context: &SessionContext, + context: &SessionContext, table_ident: &TableIdent, ) -> Result
{ let client = self.client().await?; @@ -1225,7 +1257,7 @@ impl SessionCatalog for RestSessionCatalog { .request(Method::GET, client.config.table_endpoint(table_ident)), )?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; let response = match http_response.status() { StatusCode::OK | StatusCode::NOT_MODIFIED => { @@ -1296,14 +1328,14 @@ impl SessionCatalog for RestSessionCatalog { } let client = self.client().await?; - self.check_exists_via_head(client, client.config.table_endpoint(table)) + self.check_exists_via_head(context, client, client.config.table_endpoint(table)) .await } /// Rename a table in the catalog. async fn rename_table( &self, - _context: &SessionContext, + context: &SessionContext, src: &TableIdent, dest: &TableIdent, ) -> Result<()> { @@ -1319,7 +1351,7 @@ impl SessionCatalog for RestSessionCatalog { }), )?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), @@ -1340,7 +1372,7 @@ impl SessionCatalog for RestSessionCatalog { async fn register_table( &self, - _context: &SessionContext, + context: &SessionContext, table_ident: &TableIdent, metadata_location: String, ) -> Result
{ @@ -1362,7 +1394,7 @@ impl SessionCatalog for RestSessionCatalog { }), )?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; let response: LoadTableResult = match http_response.status() { StatusCode::OK => deserialize_catalog_response::(http_response)?, @@ -1407,7 +1439,7 @@ impl SessionCatalog for RestSessionCatalog { async fn update_table( &self, - _context: &SessionContext, + context: &SessionContext, mut commit: TableCommit, ) -> Result
{ let client = self.client().await?; @@ -1426,7 +1458,7 @@ impl SessionCatalog for RestSessionCatalog { }), )?; - let http_response = client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; let response: CommitTableResponse = match http_response.status() { StatusCode::OK => deserialize_catalog_response(http_response)?, @@ -1664,7 +1696,7 @@ impl RestSessionCatalogBuilder { mod tests { use std::fs::File; use std::io::BufReader; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use chrono::{TimeZone, Utc}; use iceberg::io::LocalFsStorageFactory; @@ -1683,6 +1715,75 @@ mod tests { use crate::auth::AuthSession; use crate::request::HttpRequest; + #[derive(Debug)] + struct PlainContextSession; + + #[async_trait] + impl AuthSession for PlainContextSession { + async fn authenticate(&self, _request: &mut HttpRequest) -> Result<()> { + Ok(()) + } + } + + #[derive(Debug)] + struct ContextSession(String); + + #[async_trait] + impl AuthSession for ContextSession { + async fn authenticate(&self, request: &mut HttpRequest) -> Result<()> { + request.headers_mut().insert( + "x-session-id", + HeaderValue::from_str(&self.0).expect("valid test session ID"), + ); + Ok(()) + } + } + + #[derive(Debug)] + struct ContextManager { + catalog_session: Arc, + seen_session_ids: Arc>>, + fail_contextual_session: bool, + } + + #[async_trait] + impl AuthManager for ContextManager { + async fn init_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Box::new(PlainContextSession)) + } + + async fn catalog_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(self.catalog_session.clone()) + } + + async fn contextual_session( + &self, + context: &SessionContext, + catalog_session: Arc, + ) -> Result> { + assert!(Arc::ptr_eq(&catalog_session, &self.catalog_session)); + self.seen_session_ids + .lock() + .unwrap() + .push(context.session_id().to_string()); + if self.fail_contextual_session { + return Err(Error::new( + ErrorKind::Unexpected, + "contextual session failure", + )); + } + Ok(Arc::new(ContextSession(context.session_id().to_string()))) + } + } + fn test_catalog(config: RestCatalogConfig) -> RestSessionCatalog { RestSessionCatalog::new(config, None, None, Runtime::current(), None) } @@ -1698,6 +1799,20 @@ mod tests { ) } + fn context_catalog( + config: RestCatalogConfig, + fail_contextual_session: bool, + ) -> (RestSessionCatalog, Arc>>) { + let seen_session_ids = Arc::new(Mutex::new(Vec::new())); + let catalog_session: Arc = Arc::new(PlainContextSession); + let catalog = test_catalog_with(config, ContextManager { + catalog_session, + seen_session_ids: seen_session_ids.clone(), + fail_contextual_session, + }); + (catalog, seen_session_ids) + } + fn test_client() -> HttpClient { HttpClient::new( &RestCatalogConfig::builder() @@ -2278,7 +2393,7 @@ mod tests { bootstrap_oauth_mock.assert_async().await; // The catalog session's endpoint follows the overridden URI (visible // via the session's Debug, which prints its token endpoint). - let session_debug = format!("{:?}", client.http_client.auth_session()); + let session_debug = format!("{:?}", client.catalog_session); assert!(session_debug.contains(&format!("{}/v1/oauth/tokens", overridden.url()))); } @@ -2790,6 +2905,157 @@ mod tests { assert!(dropped.load(Ordering::SeqCst)); } + #[tokio::test] + async fn test_contextual_session_authenticates_each_catalog_request() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let first_page = server + .mock("GET", "/v1/namespaces") + .match_header("x-session-id", "session-123") + .with_body(r#"{"namespaces": [["ns1"]], "next-page-token": "next"}"#) + .create_async() + .await; + let second_page = server + .mock("GET", "/v1/namespaces?pageToken=next") + .match_header("x-session-id", "session-123") + .with_body(r#"{"namespaces": [["ns2"]]}"#) + .create_async() + .await; + + let (catalog, seen_session_ids) = context_catalog( + RestCatalogConfig::builder().uri(server.url()).build(), + false, + ); + let context = SessionContext::builder() + .session_id("session-123".to_string()) + .build(); + + let namespaces = catalog.list_namespaces(&context, None).await.unwrap(); + + assert_eq!(namespaces, vec![ + NamespaceIdent::new("ns1".to_string()), + NamespaceIdent::new("ns2".to_string()), + ]); + assert_eq!(*seen_session_ids.lock().unwrap(), vec![ + "session-123", + "session-123" + ]); + config_mock.assert_async().await; + first_page.assert_async().await; + second_page.assert_async().await; + } + + #[tokio::test] + async fn test_contextual_session_authenticates_namespace_exists_head() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock_with_exists_endpoints(&mut server).await; + let exists_mock = server + .mock("HEAD", "/v1/namespaces/ns1") + .match_header("x-session-id", "namespace-session") + .with_status(204) + .create_async() + .await; + let (catalog, seen_session_ids) = context_catalog( + RestCatalogConfig::builder().uri(server.url()).build(), + false, + ); + let context = SessionContext::builder() + .session_id("namespace-session".to_string()) + .build(); + + assert!( + catalog + .namespace_exists(&context, &NamespaceIdent::new("ns1".to_string())) + .await + .unwrap() + ); + assert_eq!(*seen_session_ids.lock().unwrap(), vec!["namespace-session"]); + config_mock.assert_async().await; + exists_mock.assert_async().await; + } + + #[tokio::test] + async fn test_contextual_session_authenticates_table_exists_head() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock_with_exists_endpoints(&mut server).await; + let exists_mock = server + .mock("HEAD", "/v1/namespaces/ns1/tables/table1") + .match_header("x-session-id", "table-session") + .with_status(204) + .create_async() + .await; + let (catalog, seen_session_ids) = context_catalog( + RestCatalogConfig::builder().uri(server.url()).build(), + false, + ); + let context = SessionContext::builder() + .session_id("table-session".to_string()) + .build(); + + assert!( + catalog + .table_exists( + &context, + &TableIdent::new(NamespaceIdent::new("ns1".to_string()), "table1".to_string(),), + ) + .await + .unwrap() + ); + assert_eq!(*seen_session_ids.lock().unwrap(), vec!["table-session"]); + config_mock.assert_async().await; + exists_mock.assert_async().await; + } + + #[tokio::test] + async fn test_contextual_session_authenticates_write_operation() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let drop_mock = server + .mock("DELETE", "/v1/namespaces/ns1") + .match_header("x-session-id", "write-session") + .with_status(204) + .create_async() + .await; + let (catalog, seen_session_ids) = context_catalog( + RestCatalogConfig::builder().uri(server.url()).build(), + false, + ); + let context = SessionContext::builder() + .session_id("write-session".to_string()) + .build(); + + catalog + .drop_namespace(&context, &NamespaceIdent::new("ns1".to_string())) + .await + .unwrap(); + assert_eq!(*seen_session_ids.lock().unwrap(), vec!["write-session"]); + config_mock.assert_async().await; + drop_mock.assert_async().await; + } + + #[tokio::test] + async fn test_contextual_session_error_prevents_operation_request() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let list_mock = server + .mock("GET", "/v1/namespaces") + .expect(0) + .create_async() + .await; + let (catalog, seen_session_ids) = + context_catalog(RestCatalogConfig::builder().uri(server.url()).build(), true); + let context = SessionContext::builder() + .session_id("failing-session".to_string()) + .build(); + + let error = catalog.list_namespaces(&context, None).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert_eq!(error.message(), "contextual session failure"); + assert_eq!(*seen_session_ids.lock().unwrap(), vec!["failing-session"]); + config_mock.assert_async().await; + list_mock.assert_async().await; + } + #[test] fn test_config_debug_redacts_secrets() { let config = RestCatalogConfig::builder() diff --git a/crates/catalog/rest/src/client.rs b/crates/catalog/rest/src/client.rs index 6de740a523..f81f86574b 100644 --- a/crates/catalog/rest/src/client.rs +++ b/crates/catalog/rest/src/client.rs @@ -17,7 +17,6 @@ use std::collections::HashMap; use std::fmt::{Debug, Formatter}; -use std::sync::Arc; use iceberg::{Error, ErrorKind, Result}; use reqwest::header::HeaderMap; @@ -25,7 +24,7 @@ use reqwest::{Client, IntoUrl, Method, RequestBuilder}; use serde::de::DeserializeOwned; use crate::RestCatalogConfig; -use crate::auth::{AuthSession, NoopSession}; +use crate::auth::AuthSession; use crate::request::HttpRequest; use crate::response::HttpResponse; @@ -41,12 +40,6 @@ pub struct HttpClient { extra_headers: HeaderMap, /// Whether to disable header redaction in error logs (defaults to false for security). disable_header_redaction: bool, - /// Authenticates everything this client sends. A client handed to an - /// [`AuthManager`] carries no authentication, since that is what the - /// manager is about to create. - /// - /// [`AuthManager`]: crate::auth::AuthManager - auth_session: Arc, } impl Debug for HttpClient { @@ -64,30 +57,13 @@ impl Debug for HttpClient { } impl HttpClient { - /// The same client authenticating with `auth_session` instead: a derived - /// auth session reuses the connection pool and headers of the client it - /// came from, carrying its own authentication. - pub fn with_auth_session(&self, auth_session: Arc) -> Self { - Self { - auth_session, - ..self.clone() - } - } - - /// The same client with no authentication, for the requests that must not - /// carry it — a session refreshing its own token over the client it - /// authenticates would otherwise recurse. - pub fn without_auth_session(&self) -> Self { - self.with_auth_session(Arc::new(NoopSession)) - } - - /// Sends a form-encoded POST and returns the response status and body, + /// Sends an unauthenticated form-encoded POST and returns the response, /// which is what an [`AuthManager`] needs to exchange a credential for a - /// token. Only `headers` are sent; the catalog's own extra headers are - /// not merged in. + /// token. /// - /// Like every request, it carries this client's session; call - /// [`Self::without_auth_session`] first to send it unauthenticated. + /// Only `headers` are sent: neither the catalog's extra headers nor + /// any session's authentication is applied, so a token request carries + /// exactly the credentials its caller puts into `headers` or `form`. /// /// [`AuthManager`]: crate::auth::AuthManager pub async fn post_form( @@ -108,7 +84,7 @@ impl HttpClient { reqwest::header::CONTENT_TYPE, reqwest::header::HeaderValue::from_static("application/x-www-form-urlencoded"), ); - self.auth_session.authenticate(&mut request).await?; + let response = self.client.execute(request.into_inner()).await?; HttpResponse::read(response).await } @@ -119,7 +95,6 @@ impl HttpClient { client: cfg.client(), extra_headers: cfg.extra_headers()?, disable_header_redaction: cfg.disable_header_redaction(), - auth_session: Arc::new(NoopSession), }) } @@ -137,28 +112,21 @@ impl HttpClient { client: cfg.client(), extra_headers, disable_header_redaction: cfg.disable_header_redaction(), - auth_session: self.auth_session, }) } - /// Testing only: the session authenticating this client's requests. - #[cfg(test)] - pub(crate) fn auth_session(&self) -> &Arc { - &self.auth_session - } - /// Testing only: the bearer token `session` would attach. /// /// Authenticates a throwaway request (never sent) and reads the header /// back, so it works for any [`AuthSession`]. #[cfg(test)] - pub(crate) async fn token(&self) -> Option { + pub(crate) async fn token(&self, auth_session: &dyn AuthSession) -> Option { let mut request = HttpRequest::build( self.client .request(Method::GET, "http://localhost/token-probe"), ) .ok()?; - self.auth_session.authenticate(&mut request).await.ok()?; + auth_session.authenticate(&mut request).await.ok()?; let request = request.into_inner(); request .headers() @@ -178,10 +146,15 @@ impl HttpClient { /// Sends `request` to the Iceberg REST catalog, authenticated by this /// client's session. - pub(crate) async fn query_catalog(&self, mut request: HttpRequest) -> Result { + pub(crate) async fn query_catalog( + &self, + auth_session: &dyn AuthSession, + mut request: HttpRequest, + ) -> Result { // Authenticate first, then apply extra headers, so a configured // `header.authorization` keeps overriding a token (unchanged behavior). - self.auth_session.authenticate(&mut request).await?; + auth_session.authenticate(&mut request).await?; + let mut request = request.into_inner(); request.headers_mut().extend(self.extra_headers.clone()); HttpResponse::read(self.client.execute(request).await?).await @@ -282,19 +255,6 @@ pub(crate) fn deserialize_unexpected_catalog_error( mod tests { use super::*; - #[derive(Debug)] - struct StaticSession; - - #[async_trait::async_trait] - impl AuthSession for StaticSession { - async fn authenticate(&self, request: &mut HttpRequest) -> Result<()> { - request - .headers_mut() - .insert("authorization", "Bearer tok".parse().unwrap()); - Ok(()) - } - } - #[tokio::test] async fn test_a_truncated_body_error_names_the_url() { // `bytes()` builds its error without a URL, so `read` attaches the one @@ -376,40 +336,36 @@ mod tests { } #[tokio::test] - async fn test_post_form_carries_the_session_until_it_is_removed() { - // Every request a client sends carries its session; a caller that - // needs an unauthenticated one removes the session first. + async fn test_post_form_is_never_authenticated() { + // A token request carries no authentication, not even a + // user-configured `header.authorization` meant for the catalog. let mut server = mockito::Server::new_async().await; - let signed = server - .mock("POST", "/token") - .match_header("authorization", "Bearer tok") - .with_status(200) - .create_async() - .await; - let unsigned = server + let mock = server .mock("POST", "/token") .match_header("authorization", mockito::Matcher::Missing) .with_status(200) .create_async() .await; - let client = HttpClient::new(&RestCatalogConfig::builder().uri(server.url()).build()) - .unwrap() - .with_auth_session(Arc::new(StaticSession)); - let url = format!("{}/token", server.url()); - - client - .post_form(&url, &HeaderMap::new(), &HashMap::new()) - .await - .unwrap(); - signed.assert_async().await; + HttpClient::new( + &RestCatalogConfig::builder() + .uri(server.url()) + .props(HashMap::from([( + "header.authorization".to_string(), + "Basic xyz".to_string(), + )])) + .build(), + ) + .unwrap() + .post_form( + &format!("{}/token", server.url()), + &HeaderMap::new(), + &HashMap::new(), + ) + .await + .unwrap(); - client - .without_auth_session() - .post_form(&url, &HeaderMap::new(), &HashMap::new()) - .await - .unwrap(); - unsigned.assert_async().await; + mock.assert_async().await; } #[test] From 49b91d1c9508758f116bdc6947d2f2cba9f9980b Mon Sep 17 00:00:00 2001 From: Danny C Jones Date: Fri, 9 Oct 2026 08:59:03 +0000 Subject: [PATCH 43/63] chore: Remove generated DEPENDENCIES.rust.tsv files, and associated scripts and guides (#3338) --- .licenserc.yaml | 2 - Makefile | 3 +- bindings/python/DEPENDENCIES.rust.tsv | 620 ------------------ crates/catalog/glue/DEPENDENCIES.rust.tsv | 429 ------------ crates/catalog/hms/DEPENDENCIES.rust.tsv | 435 ------------ crates/catalog/loader/DEPENDENCIES.rust.tsv | 492 -------------- crates/catalog/rest/DEPENDENCIES.rust.tsv | 316 --------- crates/catalog/s3tables/DEPENDENCIES.rust.tsv | 429 ------------ crates/catalog/sql/DEPENDENCIES.rust.tsv | 335 ---------- crates/examples/DEPENDENCIES.rust.tsv | 319 --------- crates/iceberg/DEPENDENCIES.rust.tsv | 311 --------- .../integration_tests/DEPENDENCIES.rust.tsv | 398 ----------- .../cache-moka/DEPENDENCIES.rust.tsv | 312 --------- crates/storage/opendal/DEPENDENCIES.rust.tsv | 522 --------------- crates/test_utils/DEPENDENCIES.rust.tsv | 311 --------- dev/release/README.md | 3 +- dev/release/dependencies.sh | 76 +-- website/src/release.md | 14 +- 18 files changed, 9 insertions(+), 5318 deletions(-) delete mode 100644 bindings/python/DEPENDENCIES.rust.tsv delete mode 100644 crates/catalog/glue/DEPENDENCIES.rust.tsv delete mode 100644 crates/catalog/hms/DEPENDENCIES.rust.tsv delete mode 100644 crates/catalog/loader/DEPENDENCIES.rust.tsv delete mode 100644 crates/catalog/rest/DEPENDENCIES.rust.tsv delete mode 100644 crates/catalog/s3tables/DEPENDENCIES.rust.tsv delete mode 100644 crates/catalog/sql/DEPENDENCIES.rust.tsv delete mode 100644 crates/examples/DEPENDENCIES.rust.tsv delete mode 100644 crates/iceberg/DEPENDENCIES.rust.tsv delete mode 100644 crates/integration_tests/DEPENDENCIES.rust.tsv delete mode 100644 crates/integrations/cache-moka/DEPENDENCIES.rust.tsv delete mode 100644 crates/storage/opendal/DEPENDENCIES.rust.tsv delete mode 100644 crates/test_utils/DEPENDENCIES.rust.tsv diff --git a/.licenserc.yaml b/.licenserc.yaml index 9959b0be40..5b3ca98e97 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -27,8 +27,6 @@ header: - "**/*.json" # Generated content by mdbook - "website/book" - # Generated content by scripts - - "**/DEPENDENCIES.*.tsv" # Generated content by cargo-public-api - "**/public-api.txt" # Generated content by trybuild diff --git a/Makefile b/Makefile index bfb0c1da87..8733abbab5 100644 --- a/Makefile +++ b/Makefile @@ -89,8 +89,7 @@ check-public-api: install-cargo-public-api check-license-notice: bash ./dev/check_license_notice.sh -# Keep version in sync with the CI lint install step in .github/workflows/ci.yml -# and EXPECTED_CARGO_DENY_VERSION in dev/release/dependencies.sh. +# Keep version in sync with the CI lint install step in .github/workflows/ci.yml. install-cargo-deny: cargo install --locked cargo-deny@0.19.9 diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv deleted file mode 100644 index 4f0b8daa21..0000000000 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,620 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 -abi_stable@0.11.3 X X -abi_stable_derive@0.11.3 X X -abi_stable_shared@0.11.0 X X -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anstream@1.0.0 X X -anstyle@1.0.14 X X -anstyle-parse@1.0.0 X X -anstyle-query@1.1.5 X X -anstyle-wincon@3.0.11 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -approx@0.5.1 X -ar_archive_writer@0.5.1 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@58.1.0 X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-csv@58.1.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-json@58.1.0 X -arrow-ord@58.3.0 X -arrow-pyarrow@58.3.0 X -arrow-row@58.1.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -as_derive_utils@0.11.0 X X -async-compression@0.4.41 X X -async-ffi@0.5.0 X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -backon@1.6.0 X -base64@0.22.1 X X -base64ct@1.8.3 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -blake2@0.10.6 X X -blake3@1.8.3 X X X -block-buffer@0.10.4 X X -block-buffer@0.12.0 X X -block-padding@0.3.3 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bstr@1.12.1 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bzip2@0.6.1 X X -cbc@0.1.2 X X -cc@1.2.57 X X -cfg-if@0.1.10 X X -cfg-if@1.0.4 X X -chacha20@0.10.0 X X -chrono@0.4.44 X X -chrono-tz@0.10.4 X X -cipher@0.4.4 X X -clap@4.6.0 X X -clap_builder@4.6.0 X X -clap_derive@4.6.0 X X -clap_lex@1.1.0 X X -cmake@0.1.57 X X -colorchoice@1.0.5 X X -colored@3.1.1 X -combine@4.6.7 X -comfy-table@7.2.2 X -compression-codecs@0.4.37 X X -compression-core@0.4.31 X X -concurrent-queue@2.5.0 X X -const-oid@0.10.2 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -const-str@1.1.0 X -const_panic@0.2.15 X -constant_time_eq@0.4.2 X X X -core-foundation@0.10.1 X X -core-foundation@0.9.4 X X -core-foundation-sys@0.8.7 X X -core_extensions@1.5.4 X X -core_extensions_proc_macros@1.5.4 X X -countio@0.3.0 X -cpufeatures@0.2.17 X X -cpufeatures@0.3.0 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -crypto-common@0.2.2 X X -csv@1.4.0 X X -csv-core@0.1.13 X X -ctor@0.6.3 X X -ctor@1.0.7 X X -ctor-proc-macro@0.0.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -datafusion@53.1.0 X -datafusion-catalog@53.1.0 X -datafusion-catalog-listing@53.1.0 X -datafusion-common@53.1.0 X -datafusion-common-runtime@53.1.0 X -datafusion-datasource@53.1.0 X -datafusion-datasource-arrow@53.1.0 X -datafusion-datasource-csv@53.1.0 X -datafusion-datasource-json@53.1.0 X -datafusion-datasource-parquet@53.1.0 X -datafusion-doc@53.1.0 X -datafusion-execution@53.1.0 X -datafusion-expr@53.1.0 X -datafusion-expr-common@53.1.0 X -datafusion-ffi@53.1.0 X -datafusion-functions@53.1.0 X -datafusion-functions-aggregate@53.1.0 X -datafusion-functions-aggregate-common@53.1.0 X -datafusion-functions-nested@53.1.0 X -datafusion-functions-table@53.1.0 X -datafusion-functions-window@53.1.0 X -datafusion-functions-window-common@53.1.0 X -datafusion-macros@53.1.0 X -datafusion-optimizer@53.1.0 X -datafusion-physical-expr@53.1.0 X -datafusion-physical-expr-adapter@53.1.0 X -datafusion-physical-expr-common@53.1.0 X -datafusion-physical-optimizer@53.1.0 X -datafusion-physical-plan@53.1.0 X -datafusion-proto@53.1.0 X -datafusion-proto-common@53.1.0 X -datafusion-pruning@53.1.0 X -datafusion-session@53.1.0 X -datafusion-sql@53.1.0 X -der@0.7.10 X X -deranged@0.5.8 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -digest@0.11.3 X X -dirs@6.0.0 X X -dirs-sys@0.5.0 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dlv-list@0.5.2 X X -dtor@0.1.1 X X -dtor-proc-macro@0.0.6 X X -dunce@1.0.5 X X X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -fixedbitset@0.5.7 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -foldhash@0.2.0 X -form_urlencoded@1.2.2 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -gearhash@0.1.3 X X -generational-arena@0.2.9 X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -git-version@0.3.9 X -git-version-macro@0.3.9 X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heapify@0.2.0 X X -heck@0.5.0 X X -hex@0.4.3 X X -hf-xet@1.5.2 X -hmac@0.12.1 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -humantime@2.3.0 X X -hybrid-array@0.4.12 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-storage-opendal@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -is_terminal_polyfill@1.70.2 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jiff-tzdb@0.1.6 X X -jiff-tzdb-platform@0.1.3 X X -jni@0.22.4 X X -jni-macros@0.22.4 X X -jni-sys@0.4.1 X X -jni-sys-macros@0.4.1 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -jsonwebtoken@10.3.0 X -konst@0.4.3 X -konst_proc_macros@0.4.1 X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libbz2-rs-sys@0.2.2 X -libc@0.2.183 X X -libloading@0.7.4 X -liblzma@0.4.6 X X -liblzma-sys@0.4.5 X X -libm@0.2.16 X -libredox@0.1.14 X -link-section@0.18.1 X X -linktime-proc-macro@0.2.0 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -matchers@0.2.0 X -md-5@0.10.6 X X -md-5@0.11.0 X X -mea@0.6.3 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -more-asserts@0.3.1 X X X X -murmur3@0.5.2 X X -ntapi@0.4.3 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-bigint-dig@0.8.6 X X -num-complex@0.4.6 X X -num-conv@0.2.0 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-traits@0.2.19 X X -objc2-core-foundation@0.3.2 X X X -objc2-io-kit@0.3.2 X X X -objc2-system-configuration@0.3.2 X X X -object@0.37.3 X X -object_store@0.13.2 X X -once_cell@1.21.4 X X -once_cell_polyfill@1.70.2 X X -oneshot@0.1.13 X X -opaque-debug@0.3.1 X X -opendal@0.57.0 X -opendal-core@0.57.0 X -opendal-layer-concurrent-limit@0.57.0 X -opendal-layer-logging@0.57.0 X -opendal-layer-retry@0.57.0 X -opendal-layer-timeout@0.57.0 X -opendal-service-azdls@0.57.0 X -opendal-service-azure-common@0.57.0 X -opendal-service-fs@0.57.0 X -opendal-service-gcs@0.57.0 X -opendal-service-hf@0.57.0 X -opendal-service-oss@0.57.0 X -opendal-service-s3@0.57.0 X -openssl-probe@0.2.1 X X -option-ext@0.2.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -ordered-multimap@0.7.3 X -os_str_bytes@6.6.1 X X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -pbkdf2@0.12.2 X X -pem@3.0.6 X -pem-rfc7468@0.7.0 X X -percent-encoding@2.3.2 X X -petgraph@0.8.3 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project@1.1.11 X X -pin-project-internal@1.1.11 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkcs1@0.7.5 X X -pkcs5@0.7.1 X X -pkcs8@0.10.2 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -prost@0.14.3 X -prost-derive@0.14.3 X -psm@0.1.30 X X -pyiceberg_core_rust@0.10.0 X -pyo3@0.28.3 X X -pyo3-build-config@0.28.3 X X -pyo3-ffi@0.28.3 X X -pyo3-macros@0.28.3 X X -pyo3-macros-backend@0.28.3 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.10.1 X X -rand@0.8.5 X X -rand@0.9.4 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.10.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redb@3.1.3 X X -redox_syscall@0.5.18 X -redox_users@0.5.2 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -repr_offset@0.2.2 X -reqsign-aliyun-oss@3.0.0 X -reqsign-aws-v4@3.0.0 X -reqsign-azure-storage@3.0.0 X -reqsign-core@3.0.0 X -reqsign-file-read-tokio@3.0.0 X -reqsign-google@3.0.0 X -reqwest@0.12.28 X X -reqwest@0.13.3 X X -reqwest-middleware@0.5.2 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rsa@0.9.10 X X -rust-ini@0.21.3 X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-platform-verifier@0.7.0 X X -rustls-platform-verifier-android@0.1.1 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -ryu@1.0.23 X X -safe-transmute@0.11.3 X -salsa20@0.10.2 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -scrypt@0.11.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sha2-asm@0.6.4 X -sharded-slab@0.1.7 X -shellexpand@3.1.2 X X -shlex@1.3.0 X X -signature@2.2.0 X X -simd-adler32@0.3.8 X -simd_cesu8@1.1.1 X X -simdutf8@0.1.5 X X -simple_asn1@0.6.4 X -siphasher@1.0.2 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -spin@0.9.8 X -spki@0.7.3 X X -sqlparser@0.61.0 X -sqlparser_derive@0.5.0 X -stable_deref_trait@1.2.1 X X -stacker@0.1.23 X X -static_assertions@1.1.0 X X -statrs@0.18.0 X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -symlink@0.1.0 X X -syn@1.0.109 X X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -sysinfo@0.38.4 X -system-configuration@0.7.0 X X -system-configuration-sys@0.6.0 X X -tagptr@0.2.0 X X -target-lexicon@0.13.5 X -tempfile@3.27.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -time@0.3.47 X X -time-core@0.1.8 X X -time-macros@0.2.27 X X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-retry@0.3.1 X -tokio-rustls@0.26.4 X X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-appender@0.2.5 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-serde@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -tstr@0.2.4 X -tstr_proc_macros@0.2.2 X -twox-hash@2.1.2 X -typed-arena@2.0.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -typewit@1.15.2 X -unicode-ident@1.0.24 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.2 X X -universal-hash@0.5.1 X X -untrusted@0.7.1 X -untrusted@0.9.0 X -url@2.5.8 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -utf8parse@0.2.2 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.7+wasi-0.2.4 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasite@1.0.2 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.5.0 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -webpki-root-certs@1.0.7 X -whoami@2.1.1 X X X -winapi@0.3.9 X X -winapi-i686-pc-windows-gnu@0.4.0 X X -winapi-util@0.1.11 X X -winapi-x86_64-pc-windows-gnu@0.4.0 X X -windows@0.62.2 X X -windows-collections@0.3.2 X X -windows-core@0.62.2 X X -windows-future@0.3.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-numerics@0.3.1 X X -windows-registry@0.6.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows-threading@0.2.1 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xattr@1.6.1 X X -xet-client@1.5.2 X -xet-core-structures@1.5.2 X -xet-data@1.5.2 X -xet-runtime@1.5.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/glue/DEPENDENCIES.rust.tsv b/crates/catalog/glue/DEPENDENCIES.rust.tsv deleted file mode 100644 index a885d1dbfc..0000000000 --- a/crates/catalog/glue/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,429 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.15 X -aws-credential-types@1.2.14 X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -aws-runtime@1.7.2 X -aws-sdk-glue@1.142.1 X -aws-sdk-sso@1.97.0 X -aws-sdk-ssooidc@1.99.0 X -aws-sdk-sts@1.101.0 X -aws-sigv4@1.4.2 X -aws-smithy-async@1.2.14 X -aws-smithy-http@0.63.6 X -aws-smithy-http-client@1.1.12 X -aws-smithy-json@0.62.5 X -aws-smithy-observability@0.2.6 X -aws-smithy-query@0.60.15 X -aws-smithy-runtime@1.10.3 X -aws-smithy-runtime-api@1.11.6 X -aws-smithy-types@1.4.7 X -aws-smithy-xml@0.60.15 X -aws-types@1.3.14 X -backon@1.6.0 X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -block-buffer@0.12.0 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bytes-utils@0.1.4 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -cmake@0.1.57 X X -combine@4.6.7 X -concurrent-queue@2.5.0 X X -const-oid@0.10.2 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -crypto-common@0.2.2 X X -ctor@1.0.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -deranged@0.5.8 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -digest@0.11.3 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dlv-list@0.5.2 X X -dunce@1.0.5 X X X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -h2@0.4.13 X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -hmac@0.12.1 X X -http@0.2.12 X X -http@1.4.0 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hybrid-array@0.4.12 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-glue@0.10.0 X -iceberg-storage-opendal@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jiff-tzdb@0.1.6 X X -jiff-tzdb-platform@0.1.3 X X -jni@0.22.4 X X -jni-macros@0.22.4 X X -jni-sys@0.4.1 X X -jni-sys-macros@0.4.1 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -link-section@0.18.1 X X -linktime-proc-macro@0.2.0 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.11.0 X X -mea@0.6.3 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.2.0 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -opendal@0.57.0 X -opendal-core@0.57.0 X -opendal-layer-concurrent-limit@0.57.0 X -opendal-layer-logging@0.57.0 X -opendal-layer-retry@0.57.0 X -opendal-layer-timeout@0.57.0 X -opendal-service-fs@0.57.0 X -opendal-service-s3@0.57.0 X -openssl-probe@0.2.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -ordered-multimap@0.7.3 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqsign-aws-v4@3.0.0 X -reqsign-core@3.0.0 X -reqsign-file-read-tokio@3.0.0 X -reqwest@0.12.28 X X -reqwest@0.13.3 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rust-ini@0.21.3 X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-platform-verifier@0.7.0 X X -rustls-platform-verifier-android@0.1.1 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.8 X X -simd-adler32@0.3.8 X -simd_cesu8@1.1.1 X X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -time@0.3.47 X X -time-core@0.1.8 X X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-rustls@0.26.4 X X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -vsimd@0.8.0 X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.5.0 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -webpki-root-certs@1.0.7 X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xattr@1.6.1 X X -xmlparser@0.13.6 X X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/hms/DEPENDENCIES.rust.tsv b/crates/catalog/hms/DEPENDENCIES.rust.tsv deleted file mode 100644 index ef7556b1bc..0000000000 --- a/crates/catalog/hms/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,435 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-broadcast@0.7.2 X X -async-lock@3.4.2 X X -async-recursion@1.1.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -block-buffer@0.12.0 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cc@1.2.57 X X -cfg-if@1.0.4 X X -cfg_aliases@0.2.1 X -chrono@0.4.44 X X -cipher@0.4.4 X X -cmake@0.1.57 X X -combine@4.6.7 X -concurrent-queue@2.5.0 X X -const-oid@0.10.2 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -crypto-common@0.2.2 X X -ctor@1.0.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -digest@0.11.3 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dlv-list@0.5.2 X X -dunce@1.0.5 X X X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -faststr@0.2.34 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -hive_metastore@0.2.0 X -hmac@0.12.1 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hybrid-array@0.4.12 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-hms@0.10.0 X -iceberg-storage-opendal@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -integer-encoding@4.1.0 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jiff-tzdb@0.1.6 X X -jiff-tzdb-platform@0.1.3 X X -jni@0.22.4 X X -jni-macros@0.22.4 X X -jni-sys@0.4.1 X X -jni-sys-macros@0.4.1 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -link-section@0.18.1 X X -linked-hash-map@0.5.6 X X -linkedbytes@0.1.16 X X -linktime-proc-macro@0.2.0 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.11.0 X X -mea@0.6.3 X -memchr@2.8.0 X X -memoffset@0.9.1 X -metainfo@0.7.14 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -motore@0.4.1 X X -motore-macros@0.4.3 X X -mur3@0.1.0 X -murmur3@0.5.2 X X -nix@0.29.0 X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -num_enum@0.7.6 X X X -num_enum_derive@0.7.6 X X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -opendal@0.57.0 X -opendal-core@0.57.0 X -opendal-layer-concurrent-limit@0.57.0 X -opendal-layer-logging@0.57.0 X -opendal-layer-retry@0.57.0 X -opendal-layer-timeout@0.57.0 X -opendal-service-fs@0.57.0 X -opendal-service-s3@0.57.0 X -openssl-probe@0.2.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -ordered-multimap@0.7.3 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pilota@0.11.10 X X -pin-project@1.1.11 X X -pin-project-internal@1.1.11 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro-crate@3.5.0 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.8.5 X X -rand@0.9.4 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -ref-cast@1.0.25 X X -ref-cast-impl@1.0.25 X X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqsign-aws-v4@3.0.0 X -reqsign-core@3.0.0 X -reqsign-file-read-tokio@3.0.0 X -reqwest@0.12.28 X X -reqwest@0.13.3 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rust-ini@0.21.3 X -rustc-hash@2.1.1 X X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-platform-verifier@0.7.0 X X -rustls-platform-verifier-android@0.1.1 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.8 X X -simd-adler32@0.3.8 X -simd_cesu8@1.1.1 X X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.5.10 X X -socket2@0.6.3 X X -sonic-number@0.1.1 X -sonic-rs@0.3.17 X -sonic-simd@0.1.3 X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.18 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-rustls@0.26.4 X X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -toml_datetime@1.0.1+spec-1.1.0 X X -toml_edit@0.25.5+spec-1.1.0 X X -toml_parser@1.0.10+spec-1.1.0 X X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -volo@0.10.7 X X -volo-thrift@0.10.8 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.5.0 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -webpki-root-certs@1.0.7 X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -winnow@1.0.0 X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xattr@1.6.1 X X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/loader/DEPENDENCIES.rust.tsv b/crates/catalog/loader/DEPENDENCIES.rust.tsv deleted file mode 100644 index 06483eafa9..0000000000 --- a/crates/catalog/loader/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,492 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-broadcast@0.7.2 X X -async-lock@3.4.2 X X -async-recursion@1.1.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.15 X -aws-credential-types@1.2.14 X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -aws-runtime@1.7.2 X -aws-sdk-glue@1.142.1 X -aws-sdk-s3tables@1.54.0 X -aws-sdk-sso@1.97.0 X -aws-sdk-ssooidc@1.99.0 X -aws-sdk-sts@1.101.0 X -aws-sigv4@1.4.2 X -aws-smithy-async@1.2.14 X -aws-smithy-http@0.63.6 X -aws-smithy-http-client@1.1.12 X -aws-smithy-json@0.62.5 X -aws-smithy-observability@0.2.6 X -aws-smithy-query@0.60.15 X -aws-smithy-runtime@1.10.3 X -aws-smithy-runtime-api@1.11.6 X -aws-smithy-types@1.4.7 X -aws-smithy-xml@0.60.15 X -aws-types@1.3.14 X -backon@1.6.0 X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -block-buffer@0.12.0 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bytes-utils@0.1.4 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -cfg_aliases@0.2.1 X -chrono@0.4.44 X X -cipher@0.4.4 X X -cmake@0.1.57 X X -combine@4.6.7 X -concurrent-queue@2.5.0 X X -const-oid@0.10.2 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc@3.4.0 X X -crc-catalog@2.4.0 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-queue@0.3.12 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -crypto-common@0.2.2 X X -ctor@1.0.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -deranged@0.5.8 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -digest@0.11.3 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dlv-list@0.5.2 X X -dunce@1.0.5 X X X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -faststr@0.2.34 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -flume@0.11.1 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -form_urlencoded@1.2.2 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-intrusive@0.5.0 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -h2@0.4.13 X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -hashlink@0.10.0 X X -heck@0.5.0 X X -hex@0.4.3 X X -hive_metastore@0.2.0 X -hmac@0.12.1 X X -http@0.2.12 X X -http@1.4.0 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hybrid-array@0.4.12 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-glue@0.10.0 X -iceberg-catalog-hms@0.10.0 X -iceberg-catalog-loader@0.10.0 X -iceberg-catalog-rest@0.10.0 X -iceberg-catalog-s3tables@0.10.0 X -iceberg-catalog-sql@0.10.0 X -iceberg-storage-opendal@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -integer-encoding@4.1.0 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jiff-tzdb@0.1.6 X X -jiff-tzdb-platform@0.1.3 X X -jni@0.22.4 X X -jni-macros@0.22.4 X X -jni-sys@0.4.1 X X -jni-sys-macros@0.4.1 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -libsqlite3-sys@0.30.1 X -link-section@0.18.1 X X -linked-hash-map@0.5.6 X X -linkedbytes@0.1.16 X X -linktime-proc-macro@0.2.0 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.11.0 X X -mea@0.6.3 X -memchr@2.8.0 X X -memoffset@0.9.1 X -metainfo@0.7.14 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -motore@0.4.1 X X -motore-macros@0.4.3 X X -mur3@0.1.0 X -murmur3@0.5.2 X X -nix@0.29.0 X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.2.0 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -num_enum@0.7.6 X X X -num_enum_derive@0.7.6 X X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -opendal@0.57.0 X -opendal-core@0.57.0 X -opendal-layer-concurrent-limit@0.57.0 X -opendal-layer-logging@0.57.0 X -opendal-layer-retry@0.57.0 X -opendal-layer-timeout@0.57.0 X -opendal-service-fs@0.57.0 X -opendal-service-s3@0.57.0 X -openssl-probe@0.2.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -ordered-multimap@0.7.3 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pilota@0.11.10 X X -pin-project@1.1.11 X X -pin-project-internal@1.1.11 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro-crate@3.5.0 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.8.5 X X -rand@0.9.4 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -ref-cast@1.0.25 X X -ref-cast-impl@1.0.25 X X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqsign-aws-v4@3.0.0 X -reqsign-core@3.0.0 X -reqsign-file-read-tokio@3.0.0 X -reqwest@0.12.28 X X -reqwest@0.13.3 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rust-ini@0.21.3 X -rustc-hash@2.1.1 X X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-platform-verifier@0.7.0 X X -rustls-platform-verifier-android@0.1.1 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.8 X X -simd-adler32@0.3.8 X -simd_cesu8@1.1.1 X X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.5.10 X X -socket2@0.6.3 X X -sonic-number@0.1.1 X -sonic-rs@0.3.17 X -sonic-simd@0.1.3 X -spin@0.9.8 X -sqlx@0.8.6 X X -sqlx-core@0.8.6 X X -sqlx-sqlite@0.8.6 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.18 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -time@0.3.47 X X -time-core@0.1.8 X X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-rustls@0.26.4 X X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -toml_datetime@1.0.1+spec-1.1.0 X X -toml_edit@0.25.5+spec-1.1.0 X X -toml_parser@1.0.10+spec-1.1.0 X X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -vcpkg@0.2.15 X X -version_check@0.9.5 X X -volo@0.10.7 X X -volo-thrift@0.10.8 X X -vsimd@0.8.0 X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.5.0 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -webpki-root-certs@1.0.7 X -webpki-roots@0.26.11 X -webpki-roots@1.0.6 X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -winnow@1.0.0 X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xattr@1.6.1 X X -xmlparser@0.13.6 X X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/rest/DEPENDENCIES.rust.tsv b/crates/catalog/rest/DEPENDENCIES.rust.tsv deleted file mode 100644 index e9a89e2b05..0000000000 --- a/crates/catalog/rest/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,316 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -h2@0.4.13 X -half@2.7.1 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-rest@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/s3tables/DEPENDENCIES.rust.tsv b/crates/catalog/s3tables/DEPENDENCIES.rust.tsv deleted file mode 100644 index a1276f6bcd..0000000000 --- a/crates/catalog/s3tables/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,429 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.15 X -aws-credential-types@1.2.14 X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -aws-runtime@1.7.2 X -aws-sdk-s3tables@1.54.0 X -aws-sdk-sso@1.97.0 X -aws-sdk-ssooidc@1.99.0 X -aws-sdk-sts@1.101.0 X -aws-sigv4@1.4.2 X -aws-smithy-async@1.2.14 X -aws-smithy-http@0.63.6 X -aws-smithy-http-client@1.1.12 X -aws-smithy-json@0.62.5 X -aws-smithy-observability@0.2.6 X -aws-smithy-query@0.60.15 X -aws-smithy-runtime@1.10.3 X -aws-smithy-runtime-api@1.11.6 X -aws-smithy-types@1.4.7 X -aws-smithy-xml@0.60.15 X -aws-types@1.3.14 X -backon@1.6.0 X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -block-buffer@0.12.0 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bytes-utils@0.1.4 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -cmake@0.1.57 X X -combine@4.6.7 X -concurrent-queue@2.5.0 X X -const-oid@0.10.2 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -crypto-common@0.2.2 X X -ctor@1.0.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -deranged@0.5.8 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -digest@0.11.3 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dlv-list@0.5.2 X X -dunce@1.0.5 X X X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -h2@0.4.13 X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -hmac@0.12.1 X X -http@0.2.12 X X -http@1.4.0 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hybrid-array@0.4.12 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-s3tables@0.10.0 X -iceberg-storage-opendal@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jiff-tzdb@0.1.6 X X -jiff-tzdb-platform@0.1.3 X X -jni@0.22.4 X X -jni-macros@0.22.4 X X -jni-sys@0.4.1 X X -jni-sys-macros@0.4.1 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -link-section@0.18.1 X X -linktime-proc-macro@0.2.0 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.11.0 X X -mea@0.6.3 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.2.0 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -opendal@0.57.0 X -opendal-core@0.57.0 X -opendal-layer-concurrent-limit@0.57.0 X -opendal-layer-logging@0.57.0 X -opendal-layer-retry@0.57.0 X -opendal-layer-timeout@0.57.0 X -opendal-service-fs@0.57.0 X -opendal-service-s3@0.57.0 X -openssl-probe@0.2.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -ordered-multimap@0.7.3 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqsign-aws-v4@3.0.0 X -reqsign-core@3.0.0 X -reqsign-file-read-tokio@3.0.0 X -reqwest@0.12.28 X X -reqwest@0.13.3 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rust-ini@0.21.3 X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-platform-verifier@0.7.0 X X -rustls-platform-verifier-android@0.1.1 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.8 X X -simd-adler32@0.3.8 X -simd_cesu8@1.1.1 X X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -time@0.3.47 X X -time-core@0.1.8 X X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-rustls@0.26.4 X X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -vsimd@0.8.0 X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.5.0 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -webpki-root-certs@1.0.7 X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xattr@1.6.1 X X -xmlparser@0.13.6 X X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/sql/DEPENDENCIES.rust.tsv b/crates/catalog/sql/DEPENDENCIES.rust.tsv deleted file mode 100644 index 7bd99de13d..0000000000 --- a/crates/catalog/sql/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,335 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc@3.4.0 X X -crc-catalog@2.4.0 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-queue@0.3.12 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -flume@0.11.1 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -form_urlencoded@1.2.2 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-intrusive@0.5.0 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -hashlink@0.10.0 X X -heck@0.5.0 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-sql@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -libsqlite3-sys@0.30.1 X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustls@0.23.37 X X X -rustls-pki-types@1.14.0 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -ryu@1.0.23 X X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -spin@0.9.8 X -sqlx@0.8.6 X X -sqlx-core@0.8.6 X X -sqlx-sqlite@0.8.6 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-stream@0.1.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -vcpkg@0.2.15 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -webpki-roots@0.26.11 X -webpki-roots@1.0.6 X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/examples/DEPENDENCIES.rust.tsv b/crates/examples/DEPENDENCIES.rust.tsv deleted file mode 100644 index 413ebda236..0000000000 --- a/crates/examples/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,319 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -h2@0.4.13 X -half@2.7.1 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-rest@0.10.0 X -iceberg-examples@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.8 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/iceberg/DEPENDENCIES.rust.tsv b/crates/iceberg/DEPENDENCIES.rust.tsv deleted file mode 100644 index 1381017ddc..0000000000 --- a/crates/iceberg/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,311 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv b/crates/integration_tests/DEPENDENCIES.rust.tsv deleted file mode 100644 index 5bb9a72a6d..0000000000 --- a/crates/integration_tests/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,398 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -block-buffer@0.12.0 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -cmake@0.1.57 X X -combine@4.6.7 X -concurrent-queue@2.5.0 X X -const-oid@0.10.2 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -crypto-common@0.2.2 X X -ctor@1.0.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -digest@0.11.3 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dlv-list@0.5.2 X X -dunce@1.0.5 X X X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -h2@0.4.13 X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -hmac@0.12.1 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hybrid-array@0.4.12 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-rest@0.10.0 X -iceberg-integration-tests@0.10.0 X -iceberg-storage-opendal@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jiff-tzdb@0.1.6 X X -jiff-tzdb-platform@0.1.3 X X -jni@0.22.4 X X -jni-macros@0.22.4 X X -jni-sys@0.4.1 X X -jni-sys-macros@0.4.1 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -link-section@0.18.1 X X -linktime-proc-macro@0.2.0 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.11.0 X X -mea@0.6.3 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -opendal@0.57.0 X -opendal-core@0.57.0 X -opendal-layer-concurrent-limit@0.57.0 X -opendal-layer-logging@0.57.0 X -opendal-layer-retry@0.57.0 X -opendal-layer-timeout@0.57.0 X -opendal-service-fs@0.57.0 X -opendal-service-s3@0.57.0 X -openssl-probe@0.2.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -ordered-multimap@0.7.3 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqsign-aws-v4@3.0.0 X -reqsign-core@3.0.0 X -reqsign-file-read-tokio@3.0.0 X -reqwest@0.12.28 X X -reqwest@0.13.3 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rust-ini@0.21.3 X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-platform-verifier@0.7.0 X X -rustls-platform-verifier-android@0.1.1 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simd_cesu8@1.1.1 X X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-rustls@0.26.4 X X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.5.0 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -webpki-root-certs@1.0.7 X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xattr@1.6.1 X X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/cache-moka/DEPENDENCIES.rust.tsv b/crates/integrations/cache-moka/DEPENDENCIES.rust.tsv deleted file mode 100644 index 9ca23a3fbc..0000000000 --- a/crates/integrations/cache-moka/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,312 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-cache-moka@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/storage/opendal/DEPENDENCIES.rust.tsv b/crates/storage/opendal/DEPENDENCIES.rust.tsv deleted file mode 100644 index 3a960f193a..0000000000 --- a/crates/storage/opendal/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,522 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anstream@1.0.0 X X -anstyle@1.0.14 X X -anstyle-parse@1.0.0 X X -anstyle-query@1.1.5 X X -anstyle-wincon@3.0.11 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -approx@0.5.1 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -backon@1.6.0 X -base64@0.22.1 X X -base64ct@1.8.3 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -blake3@1.8.3 X X X -block-buffer@0.10.4 X X -block-buffer@0.12.0 X X -block-padding@0.3.3 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bstr@1.12.1 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cbc@0.1.2 X X -cc@1.2.57 X X -cfg-if@0.1.10 X X -cfg-if@1.0.4 X X -chacha20@0.10.0 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -clap@4.6.0 X X -clap_builder@4.6.0 X X -clap_derive@4.6.0 X X -clap_lex@1.1.0 X X -cmake@0.1.57 X X -colorchoice@1.0.5 X X -colored@3.1.1 X -combine@4.6.7 X -concurrent-queue@2.5.0 X X -const-oid@0.10.2 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -const-str@1.1.0 X -const_panic@0.2.15 X -constant_time_eq@0.4.2 X X X -core-foundation@0.10.1 X X -core-foundation@0.9.4 X X -core-foundation-sys@0.8.7 X X -countio@0.3.0 X -cpufeatures@0.2.17 X X -cpufeatures@0.3.0 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -crypto-common@0.2.2 X X -csv@1.4.0 X X -csv-core@0.1.13 X X -ctor@0.6.3 X X -ctor@1.0.7 X X -ctor-proc-macro@0.0.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -der@0.7.10 X X -deranged@0.5.8 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -digest@0.11.3 X X -dirs@6.0.0 X X -dirs-sys@0.5.0 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dlv-list@0.5.2 X X -dtor@0.1.1 X X -dtor-proc-macro@0.0.6 X X -dunce@1.0.5 X X X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -gearhash@0.1.3 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -git-version@0.3.9 X -git-version-macro@0.3.9 X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heapify@0.2.0 X X -heck@0.5.0 X X -hex@0.4.3 X X -hf-xet@1.5.2 X -hmac@0.12.1 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -humantime@2.3.0 X X -hybrid-array@0.4.12 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-storage-opendal@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -is_terminal_polyfill@1.70.2 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jiff-tzdb@0.1.6 X X -jiff-tzdb-platform@0.1.3 X X -jni@0.22.4 X X -jni-macros@0.22.4 X X -jni-sys@0.4.1 X X -jni-sys-macros@0.4.1 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -jsonwebtoken@10.3.0 X -konst@0.4.3 X -konst_proc_macros@0.4.1 X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -libredox@0.1.14 X -link-section@0.18.1 X X -linktime-proc-macro@0.2.0 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -matchers@0.2.0 X -md-5@0.11.0 X X -mea@0.6.3 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -more-asserts@0.3.1 X X X X -murmur3@0.5.2 X X -ntapi@0.4.3 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-bigint-dig@0.8.6 X X -num-complex@0.4.6 X X -num-conv@0.2.0 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-traits@0.2.19 X X -objc2-core-foundation@0.3.2 X X X -objc2-io-kit@0.3.2 X X X -objc2-system-configuration@0.3.2 X X X -once_cell@1.21.4 X X -once_cell_polyfill@1.70.2 X X -oneshot@0.1.13 X X -opaque-debug@0.3.1 X X -opendal@0.57.0 X -opendal-core@0.57.0 X -opendal-layer-concurrent-limit@0.57.0 X -opendal-layer-logging@0.57.0 X -opendal-layer-retry@0.57.0 X -opendal-layer-timeout@0.57.0 X -opendal-service-azdls@0.57.0 X -opendal-service-azure-common@0.57.0 X -opendal-service-fs@0.57.0 X -opendal-service-gcs@0.57.0 X -opendal-service-hf@0.57.0 X -opendal-service-oss@0.57.0 X -opendal-service-s3@0.57.0 X -openssl-probe@0.2.1 X X -option-ext@0.2.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -ordered-multimap@0.7.3 X -os_str_bytes@6.6.1 X X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -pbkdf2@0.12.2 X X -pem@3.0.6 X -pem-rfc7468@0.7.0 X X -percent-encoding@2.3.2 X X -pin-project@1.1.11 X X -pin-project-internal@1.1.11 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkcs1@0.7.5 X X -pkcs5@0.7.1 X X -pkcs8@0.10.2 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.10.1 X X -rand@0.8.5 X X -rand@0.9.4 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.10.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redb@3.1.3 X X -redox_syscall@0.5.18 X -redox_users@0.5.2 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqsign-aliyun-oss@3.0.0 X -reqsign-aws-v4@3.0.0 X -reqsign-azure-storage@3.0.0 X -reqsign-core@3.0.0 X -reqsign-file-read-tokio@3.0.0 X -reqsign-google@3.0.0 X -reqwest@0.12.28 X X -reqwest@0.13.3 X X -reqwest-middleware@0.5.2 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rsa@0.9.10 X X -rust-ini@0.21.3 X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-platform-verifier@0.7.0 X X -rustls-platform-verifier-android@0.1.1 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -ryu@1.0.23 X X -safe-transmute@0.11.3 X -salsa20@0.10.2 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -scrypt@0.11.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sha2-asm@0.6.4 X -sharded-slab@0.1.7 X -shellexpand@3.1.2 X X -shlex@1.3.0 X X -signature@2.2.0 X X -simd-adler32@0.3.8 X -simd_cesu8@1.1.1 X X -simdutf8@0.1.5 X X -simple_asn1@0.6.4 X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -spin@0.9.8 X -spki@0.7.3 X X -stable_deref_trait@1.2.1 X X -static_assertions@1.1.0 X X -statrs@0.18.0 X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -symlink@0.1.0 X X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -sysinfo@0.38.4 X -system-configuration@0.7.0 X X -system-configuration-sys@0.6.0 X X -tagptr@0.2.0 X X -tempfile@3.27.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -time@0.3.47 X X -time-core@0.1.8 X X -time-macros@0.2.27 X X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-retry@0.3.1 X -tokio-rustls@0.26.4 X X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-appender@0.2.5 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-serde@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -typewit@1.15.2 X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.7.1 X -untrusted@0.9.0 X -url@2.5.8 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -utf8parse@0.2.2 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.7+wasi-0.2.4 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasite@1.0.2 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.5.0 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -webpki-root-certs@1.0.7 X -whoami@2.1.1 X X X -winapi@0.3.9 X X -winapi-i686-pc-windows-gnu@0.4.0 X X -winapi-util@0.1.11 X X -winapi-x86_64-pc-windows-gnu@0.4.0 X X -windows@0.62.2 X X -windows-collections@0.3.2 X X -windows-core@0.62.2 X X -windows-future@0.3.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-numerics@0.3.1 X X -windows-registry@0.6.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows-threading@0.2.1 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xattr@1.6.1 X X -xet-client@1.5.2 X -xet-core-structures@1.5.2 X -xet-data@1.5.2 X -xet-runtime@1.5.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/test_utils/DEPENDENCIES.rust.tsv b/crates/test_utils/DEPENDENCIES.rust.tsv deleted file mode 100644 index 1381017ddc..0000000000 --- a/crates/test_utils/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,311 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -array-init@2.1.0 X X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-ord@58.3.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -cipher@0.4.4 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.183 X X -libm@0.2.16 X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.52.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/dev/release/README.md b/dev/release/README.md index 93dfb4e06c..cf10a3aa3f 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -134,9 +134,8 @@ git push origin "v0.9.1" ## Dependencies -Use the dependency helper to update or verify dependency license lists: +Verify every dependency's license is one the project allows: ```shell -dev/release/dependencies.sh generate dev/release/dependencies.sh check ``` diff --git a/dev/release/dependencies.sh b/dev/release/dependencies.sh index e0d9e6a94b..36246a1cdb 100755 --- a/dev/release/dependencies.sh +++ b/dev/release/dependencies.sh @@ -25,12 +25,6 @@ if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 fi -# Keep this in sync with the cargo-deny version in the CI lint install step in -# .github/workflows/ci.yml and the install-cargo-deny target in Makefile. -# The generated DEPENDENCIES.rust.tsv files are version-sensitive, so the local -# cargo-deny must match the version CI uses to avoid spurious diffs. -EXPECTED_CARGO_DENY_VERSION="0.19.9" - CURRENT_STEP="" on_error() { @@ -48,25 +42,19 @@ trap on_error ERR usage() { cat < + $0 Commands: check Run cargo-deny license validation once at the root workspace. Default command: none; this argument is required. - generate - Regenerate each workspace package's DEPENDENCIES.rust.tsv file. - Package directories are discovered from cargo metadata. - Default command: none; this argument is required. - Options: -h, --help Show this help message. Examples: $0 check - $0 generate USAGE } @@ -112,26 +100,7 @@ require_cargo_deny() { require_command cargo if ! cargo deny --version >/dev/null 2>&1; then echo "This script requires 'cargo-deny' for dependency license checks." >&2 - echo "Install it with: cargo install --locked cargo-deny@${EXPECTED_CARGO_DENY_VERSION}" >&2 - return 1 - fi - - # The TSV output is sensitive to the cargo-deny version: different versions - # resolve the dependency graph differently, producing diffs that CI rejects. - # Assert the active binary matches the version CI pins so a shadowed or - # mismatched install fails loudly instead of generating wrong files. - local actual_version - actual_version="$(cargo deny --version | awk '{print $2}')" - if [ "${actual_version}" != "${EXPECTED_CARGO_DENY_VERSION}" ]; then - echo "ERROR: cargo-deny version mismatch." >&2 - echo " expected: ${EXPECTED_CARGO_DENY_VERSION} (pinned by CI)" >&2 - echo " found: ${actual_version} ($(command -v cargo-deny))" >&2 - echo "" >&2 - echo "Install the pinned version with:" >&2 - echo " cargo install --locked cargo-deny@${EXPECTED_CARGO_DENY_VERSION}" >&2 - echo "" >&2 - echo "If multiple cargo-deny binaries are installed, 'which -a cargo-deny'" >&2 - echo "shows which one is shadowing the others on your PATH." >&2 + echo "Install it with: cargo install --locked cargo-deny" >&2 return 1 fi } @@ -143,7 +112,7 @@ validate_args() { fi case "$1" in - check | generate) ;; + check) ;; *) usage return 1 @@ -151,23 +120,6 @@ validate_args() { esac } -discover_cargo_dirs() { - require_command cargo - require_command jq - CARGO_DIRS="$(cargo metadata \ - --format-version=1 \ - --no-deps \ - --manifest-path "${REPO_ROOT}/Cargo.toml" | - jq -r ' - .workspace_members as $workspace_members - | .packages[] - | select(.id as $id | $workspace_members | index($id)) - | .manifest_path - | sub("/Cargo.toml$"; "") - ' | - sort)" -} - check_deps_for_dir() { local cargo_dir="$1" require_cargo_deny @@ -178,32 +130,13 @@ check_deps_for_dir() { ) } -generate_deps_for_dir() { - local cargo_dir="$1" - require_cargo_deny - ( - trap - ERR - cd "${cargo_dir}" - cargo deny list -f tsv -t 0.6 > DEPENDENCIES.rust.tsv - ) -} - check_deps() { run_step "Check dependency licenses in ${REPO_ROOT}" check_deps_for_dir "${REPO_ROOT}" } -generate_deps() { - run_step "Discover Cargo workspace package directories" discover_cargo_dirs - while IFS= read -r cargo_dir; do - [ -n "${cargo_dir}" ] || continue - run_step "Generate dependency list in ${cargo_dir}" generate_deps_for_dir "${cargo_dir}" - done <<< "${CARGO_DIRS}" -} - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" COMMAND="${1:-}" -CARGO_DIRS="" show_help_if_requested "$@" run_step "Validate dependency command arguments" validate_args "$@" @@ -212,7 +145,4 @@ case "${COMMAND}" in check) check_deps ;; - generate) - generate_deps - ;; esac diff --git a/website/src/release.md b/website/src/release.md index 5e8fcad2c3..03fb665515 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -125,8 +125,8 @@ The following steps should be followed once the release is ready to begin. Dependabot runs monthly to reduce update noise, so dependencies may be out of date by the time of a release. Before cutting a release candidate, open the repository's [Dependabot page](https://github.com/apache/iceberg-rust/network/updates) and trigger a manual check for each configured ecosystem: `cargo` and `github-actions` at `/`, and `uv` at `/bindings/python`. -Review and merge the applicable dependency update PRs before creating the release branch and regenerating the dependency lists. -For an existing release branch, backport the applicable updates to that branch before regenerating its dependency lists and cutting the release candidate. +Review and merge the applicable dependency update PRs before creating the release branch. +For an existing release branch, backport the applicable updates to that branch before cutting the release candidate. Only a committer with write access to the repository can trigger Dependabot manually. Work with a committer if you do not have write access. @@ -151,7 +151,7 @@ git switch -c ${iceberg_minor_release_branch} && git push upstream ${iceberg_min - Use the minor version branch created earlier as the branch target. For example, `v0.9.x`. - Save the draft. -### Update crate versions, dependencies list, and changelog +### Update crate versions and changelog The following changes can be made in one pull request against the minor version branch (e.g. `v0.9.x`). @@ -170,13 +170,7 @@ Update `CHANGELOG.md` based on the changes since the previous version. You may use generative AI to assist making this update, but please review the proposed changes for correctness. The changelog should reflect a summary of each commit in the new release. -#### Update dependency lists - -Run the following command to update the dependencies list of every package: - -```shell -dev/release/dependencies.sh generate -``` +#### Verify dependency licenses Run the following command to verify the licenses meet the project's policy. From b0d8baaf966813f82363279db46d6688bf455d24 Mon Sep 17 00:00:00 2001 From: hylin Date: Fri, 9 Oct 2026 12:36:50 +0000 Subject: [PATCH 44/63] fix(scan): validate deletion vector delete tasks (#3145) * docs: design delete task validation * fix(scan): validate delete task construction * refactor(scan): use validated delete task builder * refactor(scan): centralize deletion vector validation * style(scan): format validated delete task changes * fix(scan): satisfy public API and license checks * test(scan): adapt merged reader fixtures * refactor(scan): encapsulate delete file metadata * fix(scan): propagate invalid delete task errors * fix(scan): unwrap the delete-file builder in the scan test `FileScanTaskDeleteFile::builder().build()` returns `Result` since this branch made it validate deletion-vector metadata, and this call site was missed, so `cargo check --all-targets` failed with E0308 once the branch was rebased onto a main that also compiles the lib test target. The task built here is a complete equality delete, so validation cannot fail; `.unwrap()` matches how the other tests in this file build tasks. * refactor(scan): make delete-file byte offsets unsigned Addresses @blackmwk's review. content_offset and content_size_in_bytes are byte quantities, so Option makes a negative unrepresentable instead of something every layer re-checks. That collapses the three findings into one change: - 'Why not change it to u64?' / 'Ditto': the getters and fields are u64. - 'This duplicates a lot.': the duplication existed only to build the 'deletion vector' vs 'delete file' prefix for two near-identical negative-value errors. Both are gone, and the remaining presence checks collapse to one lookup plus one error site. - 'I think this should also apply to non deletion vector?': it now does, and unconditionally. The i64 -> u64 conversion happens once in TryFrom<&DeleteFileContext>, where the manifest value enters, so every delete-file kind is covered rather than only Puffin position deletes. Also addresses @mbutrovich's deserialization concern from the same review: FileScanTaskDeleteFile derives plain Deserialize, so a scan plan rebuilt on a worker never passes through build(). With u64 fields, serde itself rejects a negative offset, so a hand-written or corrupted plan fails before any loader code runs -- stronger than the runtime check it replaces. Downstream, validate_deletion_vector_task loses its two u64::try_from conversions; the values are already u64. Tests: the negative cases moved from the builder to the manifest boundary (test_delete_context_rejects_negative_*), including the non-DV case, and the two loader tests now assert serde refuses the value. cargo test -p iceberg --lib: 1683 passed, 0 failed. cargo fmt --all -- --check and cargo clippy -p iceberg --all-targets -D warnings: clean. public-api.txt updated for the two getters. Signed-off-by: linhongyu510 * refactor(scan): validate delete files on deserialize and enforce size pairing Address review follow-ups on the delete-file validation design: - Route FileScanTaskDeleteFile deserialization through a FileScanTaskDeleteFileSerde mirror that builds via builder().build(), so a deletion vector rebuilt from a serialized scan plan (planner->worker path) passes validate() like a manifest- built task. A malformed plan now fails at deserialize time instead of reaching a loader expect/ok_or_else. - Enforce the spec pairing rule for every delete-file kind: content_size_in_bytes is required whenever content_offset is present, not only for deletion vectors. - Replace two builder-only duplicate tests with one driving the missing-coordinate check through get_deletes_for_data_file, where the manifest entry is converted. - Reword PR-history-laden comments to describe the code as it is now. cargo test -p iceberg --lib: 1683 passed, 0 failed. * docs(scan): clarify deletion vector loader extraction --------- Signed-off-by: linhongyu510 Co-authored-by: linhongyu510 Co-authored-by: linhongyu510 --- crates/iceberg/public-api.txt | 25 +- .../src/arrow/caching_delete_file_loader.rs | 207 ++++---- .../iceberg/src/arrow/delete_file_loader.rs | 55 +- crates/iceberg/src/arrow/delete_filter.rs | 18 +- crates/iceberg/src/arrow/reader/pipeline.rs | 5 +- .../src/arrow/reader/positional_deletes.rs | 31 +- crates/iceberg/src/arrow/reader/row_filter.rs | 23 +- crates/iceberg/src/cow_rewrite/plan.rs | 2 +- crates/iceberg/src/delete_file_index.rs | 154 +++--- crates/iceberg/src/scan/mod.rs | 5 +- crates/iceberg/src/scan/task.rs | 468 +++++++++++++++++- 11 files changed, 707 insertions(+), 286 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index bb4d350e5a..bd55395346 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1367,21 +1367,24 @@ pub fn iceberg::scan::FileScanTask::serialize(&self, serializer: S) -> core:: impl<'de> serde_core::de::Deserialize<'de> for iceberg::scan::FileScanTask pub fn iceberg::scan::FileScanTask::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::scan::FileScanTaskDeleteFile -pub iceberg::scan::FileScanTaskDeleteFile::content_offset: core::option::Option -pub iceberg::scan::FileScanTaskDeleteFile::content_size_in_bytes: core::option::Option -pub iceberg::scan::FileScanTaskDeleteFile::equality_ids: core::option::Option> -pub iceberg::scan::FileScanTaskDeleteFile::file_format: iceberg::spec::DataFileFormat -pub iceberg::scan::FileScanTaskDeleteFile::file_path: alloc::string::String -pub iceberg::scan::FileScanTaskDeleteFile::file_size_in_bytes: u64 -pub iceberg::scan::FileScanTaskDeleteFile::file_type: iceberg::spec::DataContentType -pub iceberg::scan::FileScanTaskDeleteFile::key_metadata: core::option::Option> -pub iceberg::scan::FileScanTaskDeleteFile::partition_spec_id: i32 -pub iceberg::scan::FileScanTaskDeleteFile::record_count: core::option::Option -pub iceberg::scan::FileScanTaskDeleteFile::referenced_data_file: core::option::Option +impl iceberg::scan::FileScanTaskDeleteFile +pub fn iceberg::scan::FileScanTaskDeleteFile::content_offset(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTaskDeleteFile::content_size_in_bytes(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTaskDeleteFile::equality_ids(&self) -> core::option::Option<&[i32]> +pub fn iceberg::scan::FileScanTaskDeleteFile::file_format(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::scan::FileScanTaskDeleteFile::file_path(&self) -> &str +pub fn iceberg::scan::FileScanTaskDeleteFile::file_size_in_bytes(&self) -> u64 +pub fn iceberg::scan::FileScanTaskDeleteFile::file_type(&self) -> iceberg::spec::DataContentType +pub fn iceberg::scan::FileScanTaskDeleteFile::key_metadata(&self) -> core::option::Option<&[u8]> +pub fn iceberg::scan::FileScanTaskDeleteFile::partition_spec_id(&self) -> i32 +pub fn iceberg::scan::FileScanTaskDeleteFile::record_count(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTaskDeleteFile::referenced_data_file(&self) -> core::option::Option<&str> impl core::clone::Clone for iceberg::scan::FileScanTaskDeleteFile pub fn iceberg::scan::FileScanTaskDeleteFile::clone(&self) -> iceberg::scan::FileScanTaskDeleteFile impl core::cmp::PartialEq for iceberg::scan::FileScanTaskDeleteFile pub fn iceberg::scan::FileScanTaskDeleteFile::eq(&self, other: &iceberg::scan::FileScanTaskDeleteFile) -> bool +impl core::convert::From for iceberg::Result +pub fn iceberg::Result::from(task: iceberg::scan::FileScanTaskDeleteFile) -> Self impl core::fmt::Debug for iceberg::scan::FileScanTaskDeleteFile pub fn iceberg::scan::FileScanTaskDeleteFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::scan::FileScanTaskDeleteFile diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index e1cb6de158..5582e4397f 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -263,15 +263,15 @@ impl CachingDeleteFileLoader { del_filter: DeleteFilter, schema: SchemaRef, ) -> Result { - match task.file_type { + match task.file_type() { DataContentType::PositionDeletes => { // A V3 deletion vector arrives as a PositionDeletes entry whose deletes live in // a Puffin blob, not in a positional-delete parquet file. - if task.file_format == DataFileFormat::Puffin { + if task.file_format() == DataFileFormat::Puffin { return Self::load_deletion_vector(task, basic_delete_file_loader).await; } - match del_filter.try_start_pos_del_load(&task.file_path) { + match del_filter.try_start_pos_del_load(task.file_path()) { PosDelLoadAction::AlreadyLoaded => Ok(DeleteFileContext::ExistingPosDel), PosDelLoadAction::WaitFor(notified) => { // Positional deletes are accessed synchronously by ArrowReader. @@ -281,12 +281,12 @@ impl CachingDeleteFileLoader { Ok(DeleteFileContext::ExistingPosDel) } PosDelLoadAction::Load => Ok(DeleteFileContext::PosDels { - file_path: task.file_path.clone(), + file_path: task.file_path().to_string(), stream: basic_delete_file_loader .parquet_to_batch_stream( - &task.file_path, - task.file_size_in_bytes, - task.key_metadata.as_deref(), + task.file_path(), + task.file_size_in_bytes(), + task.key_metadata(), ) .await?, }), @@ -294,22 +294,22 @@ impl CachingDeleteFileLoader { } DataContentType::EqualityDeletes => { - let Some(notify) = del_filter.try_start_eq_del_load(&task.file_path) else { + let Some(notify) = del_filter.try_start_eq_del_load(task.file_path()) else { return Ok(DeleteFileContext::ExistingEqDel); }; let (sender, receiver) = channel(); - del_filter.insert_equality_delete(&task.file_path, receiver); + del_filter.insert_equality_delete(task.file_path(), receiver); // Per the Iceberg spec, evolve schema for equality deletes but only for the // equality_ids columns, not all table columns. - let equality_ids_vec = task.equality_ids.clone().unwrap(); + let equality_ids_vec = task.equality_ids().unwrap().to_vec(); let evolved_stream = BasicDeleteFileLoader::evolve_schema( basic_delete_file_loader .parquet_to_batch_stream( - &task.file_path, - task.file_size_in_bytes, - task.key_metadata.as_deref(), + task.file_path(), + task.file_size_in_bytes(), + task.key_metadata(), ) .await?, schema, @@ -331,59 +331,47 @@ impl CachingDeleteFileLoader { } } - /// Validates a deletion-vector task and returns what the read needs as typed values: + /// Extracts a deletion-vector task's metadata as the values needed by the read: /// `(start, len, referenced data file path, expected cardinality)`. /// - /// The spec requires `referenced_data_file`, `content_offset` and `content_size_in_bytes` on - /// a deletion vector, and a deletion vector is always built from a manifest entry, so it - /// always carries `record_count`. A missing one is a manifest-entry inconsistency rather - /// than an I/O failure. - /// - /// Equality and ordinary position deletes have no equivalent validation in this loader: a - /// malformed equality/position delete file fails loudly when the Parquet reader can't open - /// it. A deletion vector's coordinates instead drive a raw byte-range read with no format - /// to fail against, so a bad coordinate would otherwise decode silently into the wrong (or - /// no) deletes, per the same corrupted-blob concern Iceberg-Java validates in - /// `BitmapPositionDeleteIndex.deserializeBitmap`. + /// Both builder construction and deserialization validate the task's required fields. + /// The fallible extraction here converts the optional fields into values for the + /// byte-range read, preserving `DataInvalid` errors instead of unwrapping them. fn validate_deletion_vector_task( task: &FileScanTaskDeleteFile, ) -> Result<(u64, u64, String, u64)> { - let content_offset = task.content_offset.ok_or_else(|| { + let content_offset = task.content_offset().ok_or_else(|| { invalid_data!( "deletion vector {} is missing content_offset", - task.file_path + task.file_path() ) })?; - let content_size = task.content_size_in_bytes.ok_or_else(|| { + let content_size = task.content_size_in_bytes().ok_or_else(|| { invalid_data!( "deletion vector {} is missing content_size_in_bytes", - task.file_path - ) - })?; - let data_file_path = task.referenced_data_file.clone().ok_or_else(|| { - invalid_data!( - "deletion vector {} is missing referenced_data_file", - task.file_path - ) - })?; - let record_count = task.record_count.ok_or_else(|| { - invalid_data!("deletion vector {} is missing record_count", task.file_path) - })?; - - let start = u64::try_from(content_offset).map_err(|_| { - invalid_data!( - "deletion vector {} has negative content_offset {content_offset}", - task.file_path + task.file_path() ) })?; - let len = u64::try_from(content_size).map_err(|_| { + let data_file_path = task + .referenced_data_file() + .map(ToOwned::to_owned) + .ok_or_else(|| { + invalid_data!( + "deletion vector {} is missing referenced_data_file", + task.file_path() + ) + })?; + let record_count = task.record_count().ok_or_else(|| { invalid_data!( - "deletion vector {} has negative content_size_in_bytes {content_size}", - task.file_path + "deletion vector {} is missing record_count", + task.file_path() ) })?; - Ok((start, len, data_file_path, record_count)) + // content_offset / content_size_in_bytes are u64 on the task, and the + // manifest -> task conversion already rejected a negative, so there is + // nothing left to convert or re-check here. + Ok((content_offset, content_size, data_file_path, record_count)) } /// Validates a decoded deletion vector's cardinality against the manifest entry's @@ -420,8 +408,8 @@ impl CachingDeleteFileLoader { let input_file = basic_delete_file_loader .file_io() - .new_input(&task.file_path)?; - let blob = match task.key_metadata.as_deref() { + .new_input(task.file_path())?; + let blob = match task.key_metadata() { Some(key_metadata) => { let key_metadata = StandardKeyMetadata::decode(key_metadata)?; EncryptedInputFile::new(input_file, key_metadata) @@ -437,7 +425,7 @@ impl CachingDeleteFileLoader { data_file_path, blob, record_count, - dv_path: task.file_path.clone(), + dv_path: task.file_path().to_string(), }) } @@ -849,9 +837,29 @@ mod tests { use super::*; use crate::arrow::delete_filter::tests::setup; use crate::scan::FileScanTaskDeleteFile; - use crate::spec::{DataContentType, Schema}; + use crate::spec::{DataContentType, DataFileFormat, Schema}; use crate::test_utils::encode_dv_blob; + #[test] + fn test_deserialized_deletion_vector_rejects_missing_fields() { + // A deletion vector that omits its required coordinates fails validation + // during deserialization (the try_from mirror routes through `build()`), + // before the loader ever sees the task. + let err = serde_json::from_value::(serde_json::json!({ + "file_path": "dv.puffin", + "file_size_in_bytes": 100, + "file_type": "PositionDeletes", + "file_format": "Puffin", + "partition_spec_id": 0 + })) + .unwrap_err(); + + assert!( + err.to_string().contains("missing referenced_data_file"), + "expected a missing-field validation error, got `{err}`" + ); + } + #[tokio::test] async fn test_delete_file_loader_parse_equality_deletes() { let tmp_dir = TempDir::new().unwrap(); @@ -1400,7 +1408,8 @@ mod tests { .with_file_type(DataContentType::PositionDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(); + .build() + .unwrap(); let eq_del = FileScanTaskDeleteFile::builder() .with_file_path(eq_delete_path.clone()) @@ -1409,7 +1418,8 @@ mod tests { .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .with_equality_ids(Some(vec![2, 3])) // Only use field IDs that exist in both schemas - .build(); + .build() + .unwrap(); let file_scan_task = FileScanTask::builder() .with_file_size_in_bytes(0) @@ -1542,8 +1552,8 @@ mod tests { dv_path: String, file_size: u64, data_file_path: String, - content_offset: i64, - content_size: i64, + content_offset: u64, + content_size: u64, record_count: u64, key_metadata: Option>, ) -> FileScanTaskDeleteFile { @@ -1559,6 +1569,7 @@ mod tests { .with_record_count(Some(record_count)) .with_key_metadata(key_metadata) .build() + .expect("deletion vector task should be valid") } #[tokio::test] @@ -1571,8 +1582,8 @@ mod tests { // Embed the blob in a Puffin-like file behind leading bytes so content_offset is // non-zero, then let the loader read it back by range. - let content_offset = 12i64; - let content_size = blob.len() as i64; + let content_offset = 12u64; + let content_size = blob.len() as u64; let mut file_bytes = vec![0u8; content_offset as usize]; file_bytes.extend_from_slice(&blob); file_bytes.extend_from_slice(&[0u8; 8]); @@ -1623,7 +1634,7 @@ mod tests { .with_aad_prefix(b"test-aad-prefix!"); let blob = encode_dv_blob([2u64, 4]); - let plaintext_size = blob.len() as i64; + let plaintext_size = blob.len() as u64; let dv_path = format!("{table_location}/deletes.puffin"); let output = EncryptedOutputFile::new(file_io.new_output(&dv_path).unwrap(), key_metadata); let file_metadata = output.write(Bytes::from(blob)).await.unwrap(); @@ -1683,7 +1694,7 @@ mod tests { std::fs::metadata(&dv_path).unwrap().len(), data_file_path, 0, - blob.len() as i64, + blob.len() as u64, 2, None, ); @@ -1708,7 +1719,8 @@ mod tests { assert!(err.message().contains("expected 2 from record_count")); } - // A well-formed deletion-vector task, for tests that then clear or corrupt one field. + // A well-formed deletion-vector task, for tests that then corrupt one field after builder + // validation. fn valid_dv_task() -> FileScanTaskDeleteFile { dv_task( "deletes.puffin".to_string(), @@ -1721,64 +1733,31 @@ mod tests { ) } + // The coordinate fields are unsigned, so a negative byte offset is rejected during + // deserialization: a hand-written or corrupted scan plan fails before any loader + // code runs. #[test] - fn test_validate_deletion_vector_task_rejects_missing_content_offset() { - let mut task = valid_dv_task(); - task.content_offset = None; - - let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("missing content_offset")); - } - - #[test] - fn test_validate_deletion_vector_task_rejects_missing_content_size() { - let mut task = valid_dv_task(); - task.content_size_in_bytes = None; - - let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("missing content_size_in_bytes")); - } - - #[test] - fn test_validate_deletion_vector_task_rejects_missing_referenced_data_file() { - let mut task = valid_dv_task(); - task.referenced_data_file = None; - - let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("missing referenced_data_file")); - } - - #[test] - fn test_validate_deletion_vector_task_rejects_missing_record_count() { - let mut task = valid_dv_task(); - task.record_count = None; - - let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("missing record_count")); - } - - #[test] - fn test_validate_deletion_vector_task_rejects_negative_content_offset() { - let mut task = valid_dv_task(); - task.content_offset = Some(-1); + fn test_deserializing_negative_content_offset_is_rejected() { + let mut task = serde_json::to_value(valid_dv_task()).unwrap(); + task["content_offset"] = serde_json::json!(-1); - let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("negative content_offset")); + let err = serde_json::from_value::(task).unwrap_err(); + assert!( + err.to_string().contains("invalid value"), + "expected an out-of-range integer error, got `{err}`" + ); } #[test] - fn test_validate_deletion_vector_task_rejects_negative_content_size() { - let mut task = valid_dv_task(); - task.content_size_in_bytes = Some(-1); + fn test_deserializing_negative_content_size_is_rejected() { + let mut task = serde_json::to_value(valid_dv_task()).unwrap(); + task["content_size_in_bytes"] = serde_json::json!(-1); - let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("negative content_size_in_bytes")); + let err = serde_json::from_value::(task).unwrap_err(); + assert!( + err.to_string().contains("invalid value"), + "expected an out-of-range integer error, got `{err}`" + ); } #[test] diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index 5852cf1acd..ec61710a42 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -124,16 +124,16 @@ impl DeleteFileLoader for BasicDeleteFileLoader { ) -> Result { let raw_batch_stream = self .parquet_to_batch_stream( - &task.file_path, - task.file_size_in_bytes, - task.key_metadata.as_deref(), + task.file_path(), + task.file_size_in_bytes(), + task.key_metadata(), ) .await?; // For equality deletes, only evolve the equality_ids columns. // For positional deletes (equality_ids is None), use all field IDs. - let field_ids = match &task.equality_ids { - Some(ids) => ids.clone(), + let field_ids = match task.equality_ids() { + Some(ids) => ids.to_vec(), None => schema.field_id_to_name_map().keys().cloned().collect(), }; @@ -230,19 +230,15 @@ mod tests { .unwrap(), ); - let task = FileScanTaskDeleteFile { - file_path: del_path.clone(), - file_size_in_bytes: std::fs::metadata(&del_path).unwrap().len(), - file_type: DataContentType::PositionDeletes, - file_format: DataFileFormat::Parquet, - partition_spec_id: 0, - equality_ids: None, - key_metadata: Some(Box::from(key_metadata.as_ref())), - referenced_data_file: None, - content_offset: None, - content_size_in_bytes: None, - record_count: None, - }; + let task = FileScanTaskDeleteFile::builder() + .with_file_path(del_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_key_metadata(Some(Box::from(key_metadata.as_ref()))) + .build() + .unwrap(); let scan_metrics = ScanMetrics::new(); let delete_file_loader = BasicDeleteFileLoader::new(file_io, scan_metrics); @@ -309,19 +305,16 @@ mod tests { .unwrap(), ); - let task = FileScanTaskDeleteFile { - file_path: del_path.clone(), - file_size_in_bytes: std::fs::metadata(&del_path).unwrap().len(), - file_type: DataContentType::EqualityDeletes, - file_format: DataFileFormat::Parquet, - partition_spec_id: 0, - equality_ids: Some(vec![1]), - key_metadata: Some(Box::from(key_metadata.as_ref())), - referenced_data_file: None, - content_offset: None, - content_size_in_bytes: None, - record_count: None, - }; + let task = FileScanTaskDeleteFile::builder() + .with_file_path(del_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(vec![1])) + .with_key_metadata(Some(Box::from(key_metadata.as_ref()))) + .build() + .unwrap(); let scan_metrics = ScanMetrics::new(); let delete_file_loader = BasicDeleteFileLoader::new(file_io, scan_metrics); diff --git a/crates/iceberg/src/arrow/delete_filter.rs b/crates/iceberg/src/arrow/delete_filter.rs index 6f13be71d2..29843295dc 100644 --- a/crates/iceberg/src/arrow/delete_filter.rs +++ b/crates/iceberg/src/arrow/delete_filter.rs @@ -208,14 +208,14 @@ impl DeleteFilter { } let Some(predicate) = self - .get_equality_delete_predicate_for_delete_file_path(&delete.file_path) + .get_equality_delete_predicate_for_delete_file_path(delete.file_path()) .await else { return Err(Error::new( ErrorKind::Unexpected, format!( "Missing predicate for equality delete file '{}'", - delete.file_path + delete.file_path() ), )); }; @@ -279,7 +279,7 @@ impl DeleteFilter { } pub(crate) fn is_equality_delete(f: &FileScanTaskDeleteFile) -> bool { - matches!(f.file_type, DataContentType::EqualityDeletes) + matches!(f.file_type(), DataContentType::EqualityDeletes) } #[cfg(test)] @@ -448,7 +448,8 @@ pub(crate) mod tests { .with_file_type(DataContentType::PositionDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(); + .build() + .unwrap(); let pos_del_2 = FileScanTaskDeleteFile::builder() .with_file_path(format!( @@ -466,7 +467,8 @@ pub(crate) mod tests { .with_file_type(DataContentType::PositionDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(); + .build() + .unwrap(); let pos_del_3 = FileScanTaskDeleteFile::builder() .with_file_path(format!( @@ -484,7 +486,8 @@ pub(crate) mod tests { .with_file_type(DataContentType::PositionDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(); + .build() + .unwrap(); let file_scan_tasks = vec![ FileScanTask::builder() @@ -561,7 +564,8 @@ pub(crate) mod tests { .with_file_type(DataContentType::EqualityDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(), + .build() + .unwrap(), ]) .with_case_sensitive(true) .build() diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 11fa200ee9..18ba42d3b9 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -2486,11 +2486,12 @@ mod tests { let delete = FileScanTaskDeleteFile::builder() .with_file_path(del_path.clone()) - .with_file_format(DataFileFormat::Parquet) .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(); + .build() + .unwrap(); let task = row_id_task_with_options(data_path, Some(100), 0, 0, vec![delete]); let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); diff --git a/crates/iceberg/src/arrow/reader/positional_deletes.rs b/crates/iceberg/src/arrow/reader/positional_deletes.rs index 2915596da7..a261bd5362 100644 --- a/crates/iceberg/src/arrow/reader/positional_deletes.rs +++ b/crates/iceberg/src/arrow/reader/positional_deletes.rs @@ -452,7 +452,8 @@ mod tests { .with_file_type(DataContentType::PositionDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(), + .build() + .unwrap(), ]) .with_case_sensitive(false) .build() @@ -672,7 +673,8 @@ mod tests { .with_file_type(DataContentType::PositionDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(), + .build() + .unwrap(), ]) .with_case_sensitive(false) .build() @@ -886,7 +888,8 @@ mod tests { .with_file_type(DataContentType::PositionDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(), + .build() + .unwrap(), ]) .with_case_sensitive(false) .build() @@ -975,8 +978,8 @@ mod tests { // Deletion vector deleting positions 1 and 3 (ids 2 and 4), serialized as a // deletion-vector-v1 blob and embedded in a Puffin-like file at a non-zero offset. let blob = encode_dv_blob([1u64, 3]); - let content_offset = 12i64; - let content_size = blob.len() as i64; + let content_offset = 12u64; + let content_size = blob.len() as u64; let mut dv_file_bytes = vec![0u8; content_offset as usize]; dv_file_bytes.extend_from_slice(&blob); let dv_path = format!("{table_location}/deletes.puffin"); @@ -1005,12 +1008,14 @@ mod tests { .with_content_offset(Some(content_offset)) .with_content_size_in_bytes(Some(content_size)) .with_record_count(Some(2)) - .build(), + .build() + .unwrap(), ]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); - let tasks = Box::pin(futures::stream::iter(vec![task])) as FileScanTaskStream; + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let result = reader .read(tasks) .unwrap() @@ -1100,14 +1105,16 @@ mod tests { .with_partition_spec_id(0) .with_referenced_data_file(Some(data_file_path.clone())) .with_content_offset(Some(0)) - .with_content_size_in_bytes(Some(blob.len() as i64)) + .with_content_size_in_bytes(Some(blob.len() as u64)) .with_record_count(Some(5)) - .build(), + .build() + .unwrap(), ]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); - let tasks = Box::pin(futures::stream::iter(vec![task])) as FileScanTaskStream; + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let result: Result, _> = reader.read(tasks).unwrap().stream().try_collect().await; diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 34b7391134..5d92f6bc7b 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -1235,19 +1235,16 @@ mod tests { .with_schema(iceberg_schema.clone()) .with_project_field_ids(vec![1, 2]) .with_predicate(Some(predicate_sub2)) - .with_deletes(vec![FileScanTaskDeleteFile { - file_path: pos_del_path.clone(), - file_type: DataContentType::PositionDeletes, - file_format: DataFileFormat::Parquet, - partition_spec_id: 0, - equality_ids: None, - file_size_in_bytes: std::fs::metadata(&pos_del_path).unwrap().len(), - referenced_data_file: None, - content_offset: None, - content_size_in_bytes: None, - record_count: None, - key_metadata: None, - }]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path(pos_del_path.clone()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_file_size_in_bytes(std::fs::metadata(&pos_del_path).unwrap().len()) + .build() + .unwrap(), + ]) .with_case_sensitive(false) .build() .unwrap(); diff --git a/crates/iceberg/src/cow_rewrite/plan.rs b/crates/iceberg/src/cow_rewrite/plan.rs index b96aea635a..8e96e9ab4b 100644 --- a/crates/iceberg/src/cow_rewrite/plan.rs +++ b/crates/iceberg/src/cow_rewrite/plan.rs @@ -151,7 +151,7 @@ mod tests { for file in files { assert_eq!(file.scan_task.deletes().len(), 1); assert_eq!( - file.scan_task.deletes()[0].file_path, + file.scan_task.deletes()[0].file_path(), format!("{}/del.parquet", fixture.table_location) ); } diff --git a/crates/iceberg/src/delete_file_index.rs b/crates/iceberg/src/delete_file_index.rs index 692ab9315a..8bae439410 100644 --- a/crates/iceberg/src/delete_file_index.rs +++ b/crates/iceberg/src/delete_file_index.rs @@ -215,25 +215,14 @@ impl PopulatedDeleteFileIndex { // A deletion vector is a position delete stored as a Puffin blob. The file // format is what distinguishes it from a position delete parquet file. if data_file.file_format() == DataFileFormat::Puffin { - // The spec requires referenced_data_file, content_offset and - // content_size_in_bytes on a deletion vector, so a missing one is a - // malformed manifest entry, not an ordinary position delete to fall back - // on. - let Some(path) = data_file.referenced_data_file() else { - return Err(invalid_data!( + // referenced_data_file is the map key the index needs; the offset/size/ + // record_count invariants are enforced centrally when the task is built. + let path = data_file.referenced_data_file().ok_or_else(|| { + invalid_data!( "deletion vector {} is missing referenced_data_file", - arc_ctx.manifest_entry.file_path() - )); - }; - - if data_file.content_offset().is_none() - || data_file.content_size_in_bytes().is_none() - { - return Err(invalid_data!( - "deletion vector {} is missing content_offset or content_size_in_bytes", - arc_ctx.manifest_entry.file_path() - )); - } + data_file.file_path() + ) + })?; if let Some(existing) = dvs_by_referenced_data_file.insert(path.clone(), arc_ctx) @@ -295,27 +284,25 @@ impl PopulatedDeleteFileIndex { ) -> Result> { let mut results = vec![]; - self.global_equality_deletes - .iter() + for delete in self.global_equality_deletes.iter().filter(|&delete| { // filter that returns true if the provided delete file's sequence number is **greater than** `seq_num` - .filter(|&delete| { - seq_num - .map(|seq_num| delete.manifest_entry.sequence_number() > Some(seq_num)) - .unwrap_or_else(|| true) - }) - .for_each(|delete| results.push(delete.as_ref().into())); + seq_num + .map(|seq_num| delete.manifest_entry.sequence_number() > Some(seq_num)) + .unwrap_or_else(|| true) + }) { + results.push(FileScanTaskDeleteFile::try_from(delete.as_ref())?); + } if let Some(deletes) = self.eq_deletes_by_partition.get(data_file.partition()) { - deletes - .iter() + for delete in deletes.iter().filter(|&delete| { // filter that returns true if the provided delete file's sequence number is **greater than** `seq_num` - .filter(|&delete| { - seq_num - .map(|seq_num| delete.manifest_entry.sequence_number() > Some(seq_num)) - .unwrap_or_else(|| true) - && data_file.partition_spec_id == delete.partition_spec_id - }) - .for_each(|delete| results.push(delete.as_ref().into())); + seq_num + .map(|seq_num| delete.manifest_entry.sequence_number() > Some(seq_num)) + .unwrap_or_else(|| true) + && data_file.partition_spec_id == delete.partition_spec_id + }) { + results.push(FileScanTaskDeleteFile::try_from(delete.as_ref())?); + } } // A deletion vector supersedes all position delete files for its data file, per the spec: @@ -351,12 +338,12 @@ impl PopulatedDeleteFileIndex { )); } } - results.push(dv.as_ref().into()); + results.push(FileScanTaskDeleteFile::try_from(dv.as_ref())?); return Ok(results); } if let Some(deletes) = self.pos_deletes_by_partition.get(data_file.partition()) { - deletes + for delete in deletes .iter() // filter that returns true if the provided delete file's sequence number is **greater than or equal to** `seq_num` .filter(|&delete| { @@ -365,14 +352,16 @@ impl PopulatedDeleteFileIndex { .unwrap_or_else(|| true) && data_file.partition_spec_id == delete.partition_spec_id }) - .for_each(|delete| results.push(delete.as_ref().into())); + { + results.push(FileScanTaskDeleteFile::try_from(delete.as_ref())?); + } } // Position deletes indexed by the exact path of the data file they reference. // An exact path match is sufficient proof that the delete applies, so no // partition spec id check is performed. if let Some(deletes) = self.pos_deletes_by_path.get(data_file.file_path()) { - deletes + for delete in deletes .iter() // filter that returns true if the provided delete file's sequence number is **greater than or equal to** `seq_num` .filter(|&delete| { @@ -380,7 +369,9 @@ impl PopulatedDeleteFileIndex { .map(|seq_num| delete.manifest_entry.sequence_number() >= Some(seq_num)) .unwrap_or(true) }) - .for_each(|delete| results.push(delete.as_ref().into())); + { + results.push(FileScanTaskDeleteFile::try_from(delete.as_ref())?); + } } Ok(results) @@ -442,7 +433,7 @@ mod tests { .unwrap(); let actual_paths_to_apply_for_seq_4: Vec = delete_files_to_apply_for_seq_4 .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert_eq!( @@ -456,7 +447,7 @@ mod tests { .unwrap(); let actual_paths_to_apply_for_seq_5: Vec = delete_files_to_apply_for_seq_5 .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert_eq!( actual_paths_to_apply_for_seq_5, @@ -469,7 +460,7 @@ mod tests { .unwrap(); let actual_paths_to_apply_for_seq_6: Vec = delete_files_to_apply_for_seq_6 .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert_eq!( actual_paths_to_apply_for_seq_6, @@ -486,7 +477,7 @@ mod tests { let actual_paths_to_apply_for_partitioned_file: Vec = delete_files_to_apply_for_partitioned_file .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert_eq!( actual_paths_to_apply_for_partitioned_file, @@ -541,7 +532,7 @@ mod tests { .unwrap(); let actual_paths_to_apply_for_seq_4: Vec = delete_files_to_apply_for_seq_4 .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert_eq!( @@ -555,7 +546,7 @@ mod tests { .unwrap(); let actual_paths_to_apply_for_seq_5: Vec = delete_files_to_apply_for_seq_5 .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert_eq!( actual_paths_to_apply_for_seq_5, @@ -568,7 +559,7 @@ mod tests { .unwrap(); let actual_paths_to_apply_for_seq_6: Vec = delete_files_to_apply_for_seq_6 .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert_eq!( actual_paths_to_apply_for_seq_6, @@ -584,7 +575,7 @@ mod tests { let actual_paths_to_apply_for_different_partition: Vec = delete_files_to_apply_for_different_partition .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert!(actual_paths_to_apply_for_different_partition.is_empty()); @@ -596,7 +587,7 @@ mod tests { let actual_paths_to_apply_for_different_spec: Vec = delete_files_to_apply_for_different_spec .into_iter() - .map(|file| file.file_path) + .map(|file| file.file_path().to_string()) .collect(); assert!(actual_paths_to_apply_for_different_spec.is_empty()); } @@ -617,7 +608,7 @@ mod tests { .get_deletes_for_data_file(&data_file_a, Some(0)) .unwrap(); assert_eq!(deletes_for_a.len(), 1); - assert_eq!(deletes_for_a[0].file_path, pos_delete.file_path()); + assert_eq!(deletes_for_a[0].file_path(), pos_delete.file_path()); // The delete references data file A, so it must not apply to data file B // even though B shares A's partition. @@ -785,7 +776,7 @@ mod tests { .get_deletes_for_data_file(&data_file, Some(0)) .unwrap() .into_iter() - .map(|delete| delete.file_path) + .map(|delete| delete.file_path().to_string()) .collect(); actual_paths.sort(); @@ -822,7 +813,7 @@ mod tests { .get_deletes_for_data_file(&data_file, Some(0)) .unwrap() .into_iter() - .map(|delete| delete.file_path) + .map(|delete| delete.file_path().to_string()) .collect(); actual_paths.sort(); @@ -861,7 +852,10 @@ mod tests { .get_deletes_for_data_file(&data_file, Some(0)) .unwrap(); assert_eq!(deletes_for_referenced.len(), 1); - assert_eq!(deletes_for_referenced[0].file_path, pos_delete.file_path()); + assert_eq!( + deletes_for_referenced[0].file_path(), + pos_delete.file_path() + ); assert!( index @@ -964,13 +958,13 @@ mod tests { partition_spec_id: 0, }; - let task: FileScanTaskDeleteFile = (&ctx).into(); - assert_eq!(task.file_type, DataContentType::PositionDeletes); - assert_eq!(task.content_offset, Some(4)); - assert_eq!(task.content_size_in_bytes, Some(40)); - assert_eq!(task.record_count, Some(3)); + let task = FileScanTaskDeleteFile::try_from(&ctx).unwrap(); + assert_eq!(task.file_type(), DataContentType::PositionDeletes); + assert_eq!(task.content_offset(), Some(4)); + assert_eq!(task.content_size_in_bytes(), Some(40)); + assert_eq!(task.record_count(), Some(3)); assert_eq!( - task.referenced_data_file.as_deref(), + task.referenced_data_file(), Some("s3://bucket/data/part-0.parquet") ); } @@ -1001,7 +995,7 @@ mod tests { .get_deletes_for_data_file(&data_file, Some(0)) .unwrap() .into_iter() - .map(|f| f.file_path) + .map(|f| f.file_path().to_string()) .collect(); // Only the deletion vector applies; the partition-scoped position delete file, which @@ -1116,7 +1110,7 @@ mod tests { .get_deletes_for_data_file(&data_file, Some(0)) .unwrap() .into_iter() - .map(|f| f.file_path) + .map(|f| f.file_path().to_string()) .collect(); // Only the deletion vector applies; the path-scoped position delete file, which would @@ -1151,7 +1145,7 @@ mod tests { .get_deletes_for_data_file(&data_file, Some(0)) .unwrap() .into_iter() - .map(|f| f.file_path) + .map(|f| f.file_path().to_string()) .collect(); applied.sort(); @@ -1162,7 +1156,7 @@ mod tests { } #[test] - fn test_deletion_vector_missing_referenced_data_file_is_rejected() { + fn test_deletion_vector_index_rejects_missing_referenced_data_file() { let malformed_dv = DataFileBuilder::default() .file_path("deletes.puffin".to_string()) .file_format(DataFileFormat::Puffin) @@ -1175,43 +1169,59 @@ mod tests { .file_size_in_bytes(60) .build() .unwrap(); - let contexts = vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &malformed_dv).into(), partition_spec_id: 0, }]; let err = PopulatedDeleteFileIndex::new(contexts).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!(err.message().contains("missing referenced_data_file")); } #[test] - fn test_deletion_vector_missing_coordinates_is_rejected() { + fn test_deletion_vector_index_rejects_missing_coordinates() { + // The missing-coordinate check fires in `get_deletes_for_data_file`, where the + // manifest entry is converted into a task. `PopulatedDeleteFileIndex::new` only + // needs `referenced_data_file` as its map key, so a DV that carries it but omits + // `content_offset` survives construction and is rejected when the data file is + // resolved against it. + let data_file = DataFileBuilder::default() + .file_path("target-data.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .content(DataContentType::Data) + .record_count(100) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(100) + .build() + .unwrap(); + let malformed_dv = DataFileBuilder::default() .file_path("deletes.puffin".to_string()) .file_format(DataFileFormat::Puffin) .content(DataContentType::PositionDeletes) .record_count(1) - .referenced_data_file(Some("data.parquet".to_string())) + .referenced_data_file(Some("target-data.parquet".to_string())) .content_size_in_bytes(Some(40)) .partition(Struct::empty()) .partition_spec_id(0) .file_size_in_bytes(60) .build() .unwrap(); - let contexts = vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &malformed_dv).into(), partition_spec_id: 0, }]; - let err = PopulatedDeleteFileIndex::new(contexts).unwrap_err(); + let delete_file_index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let err = delete_file_index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!( - err.message() - .contains("missing content_offset or content_size_in_bytes") - ); + assert!(err.message().contains("missing content_offset")); } #[test] diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index ccdb5c7d5b..0c241125dc 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -1881,7 +1881,8 @@ mod tests { .with_content_size_in_bytes(Some(34)) .with_record_count(Some(5)) .with_key_metadata(Some(vec![4, 5, 6].into_boxed_slice())) - .build(), + .build() + .unwrap(), ]) .with_partition(Some(Struct::from_iter([Some(Literal::long(42))]))) .with_partition_spec(Some(partition_spec)) @@ -2905,7 +2906,7 @@ mod tests { "positional delete file should be planned into the task" ); assert_eq!( - tasks[0].deletes()[0].file_type, + tasks[0].deletes()[0].file_type(), DataContentType::PositionDeletes ); diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 97ca7e5181..45940b0dfd 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -21,13 +21,13 @@ use futures::stream::BoxStream; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; -use crate::Result; use crate::error::invalid_data; use crate::expr::BoundPredicate; use crate::spec::{ DataContentType, DataFileFormat, ManifestEntryRef, NameMapping, PartitionSpec, Schema, SchemaRef, SortOrderRef, Struct, StructType, }; +use crate::{Error, Result}; /// A stream of [`FileScanTask`]. pub type FileScanTaskStream = BoxStream<'static, Result>; @@ -309,8 +309,24 @@ pub(crate) struct DeleteFileContext { pub(crate) partition_spec_id: i32, } -impl From<&DeleteFileContext> for FileScanTaskDeleteFile { - fn from(ctx: &DeleteFileContext) -> Self { +impl TryFrom<&DeleteFileContext> for FileScanTaskDeleteFile { + type Error = Error; + + fn try_from(ctx: &DeleteFileContext) -> Result { + // The manifest stores these as i64. Convert here so a negative byte offset is + // rejected once at the boundary, for every delete-file kind rather than only + // deletion vectors, instead of being carried inward and re-checked per consumer. + let file_path = ctx.manifest_entry.file_path(); + let to_offset = |value: Option, field: &str| -> Result> { + value + .map(|value| { + u64::try_from(value).map_err(|_| { + invalid_data!("delete file {file_path} has negative {field} {value}") + }) + }) + .transpose() + }; + FileScanTaskDeleteFile::builder() .with_file_path(ctx.manifest_entry.file_path().to_string()) .with_file_size_in_bytes(ctx.manifest_entry.file_size_in_bytes()) @@ -319,8 +335,14 @@ impl From<&DeleteFileContext> for FileScanTaskDeleteFile { .with_partition_spec_id(ctx.partition_spec_id) .with_equality_ids(ctx.manifest_entry.data_file.equality_ids.clone()) .with_referenced_data_file(ctx.manifest_entry.data_file.referenced_data_file.clone()) - .with_content_offset(ctx.manifest_entry.data_file.content_offset) - .with_content_size_in_bytes(ctx.manifest_entry.data_file.content_size_in_bytes) + .with_content_offset(to_offset( + ctx.manifest_entry.data_file.content_offset, + "content_offset", + )?) + .with_content_size_in_bytes(to_offset( + ctx.manifest_entry.data_file.content_size_in_bytes, + "content_size_in_bytes", + )?) .with_record_count(Some(ctx.manifest_entry.record_count())) .with_key_metadata( ctx.manifest_entry @@ -335,55 +357,62 @@ impl From<&DeleteFileContext> for FileScanTaskDeleteFile { /// A task to scan part of file. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TypedBuilder)] -#[builder(field_defaults(setter(prefix = "with_")))] +#[serde(try_from = "crate::scan::task::_serde::FileScanTaskDeleteFileSerde")] +#[builder( + field_defaults(setter(prefix = "with_")), + build_method(into = Result) +)] pub struct FileScanTaskDeleteFile { /// The delete file path - pub file_path: String, + file_path: String, /// The total size of the delete file in bytes, from the manifest entry. - pub file_size_in_bytes: u64, + file_size_in_bytes: u64, /// delete file type - pub file_type: DataContentType, + file_type: DataContentType, /// The delete file's format, from the manifest entry. A `PositionDeletes` entry written as /// `Puffin` is a V3 deletion vector; one written as `Parquet` is a position delete file. - pub file_format: DataFileFormat, + file_format: DataFileFormat, /// partition id - pub partition_spec_id: i32, + partition_spec_id: i32, /// equality ids for equality deletes (null for anything other than equality-deletes) #[builder(default)] - pub equality_ids: Option>, + equality_ids: Option>, /// For a deletion vector, the location of the data file whose rows it deletes. Required for /// deletion vectors, and may also be set on a position delete file scoped to one data file. #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub referenced_data_file: Option, + referenced_data_file: Option, /// For a deletion vector, the offset of the blob within its Puffin file. Set only for - /// deletion vectors, where it locates the blob for direct access. + /// deletion vectors, where it locates the blob for direct access. Unsigned because it is a + /// byte offset: a negative value is rejected once, when the task is built from a manifest + /// entry, rather than re-checked by every consumer. #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub content_offset: Option, + content_offset: Option, /// For a deletion vector, the length in bytes of the blob within its Puffin file. /// Required together with `content_offset`; both are absent for non-DV delete files. + /// Unsigned for the same reason as `content_offset`. #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub content_size_in_bytes: Option, + content_size_in_bytes: Option, /// The number of records in the delete file, from the manifest entry; for a deletion vector, /// the cardinality of its bitmap. `None` only for a task not built from a manifest entry. #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub record_count: Option, + record_count: Option, /// Key metadata for an encrypted delete file. When present, the reader uses this to /// decrypt the file: for a Parquet equality or position delete file, this builds @@ -397,7 +426,7 @@ pub struct FileScanTaskDeleteFile { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub key_metadata: Option>, + key_metadata: Option>, } mod _serde { @@ -409,8 +438,8 @@ mod _serde { use crate::error::invalid_data; use crate::expr::BoundPredicate; use crate::spec::{ - DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral, SchemaRef, SortOrderRef, - StructType, Type, + DataContentType, DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral, + SchemaRef, SortOrderRef, StructType, Type, }; use crate::{Error, Result}; @@ -579,13 +608,168 @@ mod _serde { .build() } } + + #[derive(Deserialize)] + pub(super) struct FileScanTaskDeleteFileSerde { + file_path: String, + file_size_in_bytes: u64, + file_type: DataContentType, + file_format: DataFileFormat, + partition_spec_id: i32, + equality_ids: Option>, + #[serde(default)] + referenced_data_file: Option, + #[serde(default)] + content_offset: Option, + #[serde(default)] + content_size_in_bytes: Option, + #[serde(default)] + record_count: Option, + #[serde(default)] + key_metadata: Option>, + } + + impl TryFrom for FileScanTaskDeleteFile { + type Error = Error; + + fn try_from(value: FileScanTaskDeleteFileSerde) -> Result { + Self::builder() + .with_file_path(value.file_path) + .with_file_size_in_bytes(value.file_size_in_bytes) + .with_file_type(value.file_type) + .with_file_format(value.file_format) + .with_partition_spec_id(value.partition_spec_id) + .with_equality_ids(value.equality_ids) + .with_referenced_data_file(value.referenced_data_file) + .with_content_offset(value.content_offset) + .with_content_size_in_bytes(value.content_size_in_bytes) + .with_record_count(value.record_count) + .with_key_metadata(value.key_metadata) + .build() + } + } +} + +impl FileScanTaskDeleteFile { + /// Returns the delete file path. + pub fn file_path(&self) -> &str { + &self.file_path + } + + /// Returns the total size of the delete file in bytes. + pub fn file_size_in_bytes(&self) -> u64 { + self.file_size_in_bytes + } + + /// Returns the delete file content type. + pub fn file_type(&self) -> DataContentType { + self.file_type + } + + /// Returns the delete file format. + pub fn file_format(&self) -> DataFileFormat { + self.file_format + } + + /// Returns the partition spec id. + pub fn partition_spec_id(&self) -> i32 { + self.partition_spec_id + } + + /// Returns the equality field ids for an equality delete file. + pub fn equality_ids(&self) -> Option<&[i32]> { + self.equality_ids.as_deref() + } + + /// Returns the referenced data file path. + pub fn referenced_data_file(&self) -> Option<&str> { + self.referenced_data_file.as_deref() + } + + /// Returns the deletion vector blob offset. + pub fn content_offset(&self) -> Option { + self.content_offset + } + + /// Returns the deletion vector blob size in bytes. + pub fn content_size_in_bytes(&self) -> Option { + self.content_size_in_bytes + } + + /// Returns the number of records in the delete file. + pub fn record_count(&self) -> Option { + self.record_count + } + + /// Returns the key metadata for the encrypted delete file. + pub fn key_metadata(&self) -> Option<&[u8]> { + self.key_metadata.as_deref() + } + + fn is_deletion_vector(&self) -> bool { + self.file_type == DataContentType::PositionDeletes + && self.file_format == DataFileFormat::Puffin + } + + fn validate(&self) -> Result<()> { + // Negative offsets/sizes are unrepresentable: both fields are u64, and the + // conversion from the manifest entry's i64 rejects a negative there. + + // Spec: `content_size_in_bytes` is required whenever `content_offset` is + // present, for every delete-file kind (not only deletion vectors). + if self.content_offset.is_some() && self.content_size_in_bytes.is_none() { + return Err(invalid_data!( + "delete file {} is missing content_size_in_bytes while content_offset is present", + self.file_path + )); + } + + // Presence of the deletion-vector coordinates is required only for + // deletion vectors; other delete files legitimately carry none of these. + if !self.is_deletion_vector() { + return Ok(()); + } + + let missing = if self.referenced_data_file.is_none() { + Some("referenced_data_file") + } else if self.content_offset.is_none() { + Some("content_offset") + } else if self.content_size_in_bytes.is_none() { + Some("content_size_in_bytes") + } else if self.record_count.is_none() { + Some("record_count") + } else { + None + }; + + if let Some(field) = missing { + return Err(invalid_data!( + "deletion vector {} is missing {field}", + self.file_path + )); + } + + Ok(()) + } +} + +impl From for Result { + fn from(task: FileScanTaskDeleteFile) -> Self { + task.validate()?; + Ok(task) + } } #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::*; use crate::ErrorKind; - use crate::spec::{Literal, NestedField, PrimitiveType, Transform, Type}; + use crate::spec::{ + DataFileBuilder, Literal, ManifestEntry, ManifestStatus, NestedField, PrimitiveType, + Transform, Type, + }; fn build_file_scan_task( schema: SchemaRef, @@ -630,6 +814,34 @@ mod tests { (schema, partition_spec) } + fn build_delete_file_task( + file_type: DataContentType, + file_format: DataFileFormat, + ) -> Result { + FileScanTaskDeleteFile::builder() + .with_file_path("delete-file".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(file_type) + .with_file_format(file_format) + .with_partition_spec_id(0) + .build() + } + + fn assert_delete_file_builder_error( + result: Result, + expected_message: &str, + ) { + match result { + Ok(task) => panic!( + "expected delete file builder to fail with `{expected_message}`, but got Ok({task:?})" + ), + Err(err) => { + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), expected_message); + } + } + } + #[test] fn test_file_scan_task_builder_rejects_non_empty_partition_without_spec() { // Regression test for https://github.com/apache/iceberg-rust/issues/3130. @@ -739,4 +951,218 @@ mod tests { assert_eq!(err.kind(), ErrorKind::DataInvalid); } + + #[test] + fn test_delete_file_builder_accepts_valid_deletion_vector() { + let task = FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(7) + .with_referenced_data_file(Some("data.parquet".to_string())) + .with_content_offset(Some(11)) + .with_content_size_in_bytes(Some(13)) + .with_record_count(Some(3)) + .with_key_metadata(Some(vec![17, 19].into_boxed_slice())) + .build() + .unwrap(); + + assert_eq!(task.file_path(), "dv.puffin"); + assert_eq!(task.file_size_in_bytes(), 100); + assert_eq!(task.file_type(), DataContentType::PositionDeletes); + assert_eq!(task.file_format(), DataFileFormat::Puffin); + assert_eq!(task.partition_spec_id(), 7); + assert_eq!(task.equality_ids(), None); + assert_eq!(task.referenced_data_file(), Some("data.parquet")); + assert_eq!(task.content_offset(), Some(11)); + assert_eq!(task.content_size_in_bytes(), Some(13)); + assert_eq!(task.record_count(), Some(3)); + assert_eq!(task.key_metadata(), Some([17, 19].as_slice())); + } + + #[test] + fn test_delete_file_builder_rejects_dv_missing_referenced_data_file() { + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_content_offset(Some(7)) + .with_content_size_in_bytes(Some(11)) + .with_record_count(Some(3)) + .build(), + "deletion vector dv.puffin is missing referenced_data_file", + ); + } + + #[test] + fn test_delete_file_builder_rejects_dv_missing_content_offset() { + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some("data.parquet".to_string())) + .with_content_size_in_bytes(Some(11)) + .with_record_count(Some(3)) + .build(), + "deletion vector dv.puffin is missing content_offset", + ); + } + + #[test] + fn test_delete_file_builder_rejects_dv_missing_content_size() { + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some("data.parquet".to_string())) + .with_content_offset(Some(7)) + .with_record_count(Some(3)) + .build(), + "delete file dv.puffin is missing content_size_in_bytes while content_offset is present", + ); + } + + #[test] + fn test_delete_file_builder_rejects_dv_missing_record_count() { + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some("data.parquet".to_string())) + .with_content_offset(Some(7)) + .with_content_size_in_bytes(Some(11)) + .build(), + "deletion vector dv.puffin is missing record_count", + ); + } + + #[test] + fn test_delete_file_builder_rejects_non_dv_offset_without_size() { + // Spec pairing rule applies to every delete-file kind: a Parquet + // position-delete file that carries a content_offset must also carry a + // content_size_in_bytes, even though it is not a deletion vector. + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("position-deletes.parquet".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_content_offset(Some(7)) + .build(), + "delete file position-deletes.parquet is missing content_size_in_bytes while content_offset is present", + ); + } + + fn delete_manifest_entry( + file_path: &str, + file_format: DataFileFormat, + content_offset: Option, + content_size_in_bytes: Option, + ) -> DeleteFileContext { + let mut data_file = DataFileBuilder::default() + .content(DataContentType::PositionDeletes) + .file_path(file_path.to_string()) + .file_format(file_format) + .partition(Struct::empty()) + .record_count(3) + .file_size_in_bytes(100) + .column_sizes(HashMap::new()) + .value_counts(HashMap::new()) + .null_value_counts(HashMap::new()) + .partition_spec_id(0) + .referenced_data_file(Some("data.parquet".to_string())) + .build() + .unwrap(); + data_file.content_offset = content_offset; + data_file.content_size_in_bytes = content_size_in_bytes; + + DeleteFileContext { + manifest_entry: Arc::new( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(data_file) + .build(), + ), + partition_spec_id: 0, + } + } + + fn assert_delete_context_error(ctx: &DeleteFileContext, expected_message: &str) { + match FileScanTaskDeleteFile::try_from(ctx) { + Ok(task) => panic!( + "expected conversion to fail with `{expected_message}`, but got Ok({task:?})" + ), + Err(error) => { + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!( + error.message().contains(expected_message), + "expected `{expected_message}`, got `{}`", + error.message() + ); + } + } + } + + /// The manifest entry supplies `content_offset` / `content_size_in_bytes` as + /// `i64`; the conversion into a task rejects a negative value as it enters, + /// since the task fields are `u64`. + #[test] + fn test_delete_context_rejects_negative_dv_offset() { + assert_delete_context_error( + &delete_manifest_entry("dv.puffin", DataFileFormat::Puffin, Some(-1), Some(11)), + "delete file dv.puffin has negative content_offset -1", + ); + } + + #[test] + fn test_delete_context_rejects_negative_dv_size() { + assert_delete_context_error( + &delete_manifest_entry("dv.puffin", DataFileFormat::Puffin, Some(7), Some(-1)), + "delete file dv.puffin has negative content_size_in_bytes -1", + ); + } + + /// The negative-coordinate check applies to every delete-file kind, including + /// a Parquet position-delete file rather than only deletion vectors. + #[test] + fn test_delete_context_rejects_negative_coordinates_for_non_dv() { + assert_delete_context_error( + &delete_manifest_entry( + "position-deletes.parquet", + DataFileFormat::Parquet, + Some(-1), + None, + ), + "delete file position-deletes.parquet has negative content_offset -1", + ); + assert_delete_context_error( + &delete_manifest_entry( + "position-deletes.parquet", + DataFileFormat::Parquet, + None, + Some(-1), + ), + "delete file position-deletes.parquet has negative content_size_in_bytes -1", + ); + } + + #[test] + fn test_delete_file_builder_accepts_non_dv_delete_without_dv_fields() { + build_delete_file_task(DataContentType::PositionDeletes, DataFileFormat::Parquet).unwrap(); + build_delete_file_task(DataContentType::EqualityDeletes, DataFileFormat::Parquet).unwrap(); + } } From 5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Fri, 9 Oct 2026 16:08:16 +0000 Subject: [PATCH 45/63] perf(manifest): upgrade to apache-avro 0.22 and read manifest entries without Value (#3354) * perf(manifest): upgrade to apache-avro 0.22 and read manifest entries without Value * add test cases, update docs * address PR feedback * address PR feedback. * address PR feedback --- Cargo.lock | 110 +- Cargo.toml | 2 +- crates/iceberg/Cargo.toml | 1 + crates/iceberg/src/avro/deserializer.rs | 457 ++++++++ crates/iceberg/src/avro/mod.rs | 6 + crates/iceberg/src/avro/named_types.rs | 309 ++++++ crates/iceberg/src/avro/schema.rs | 158 ++- crates/iceberg/src/encryption/key_metadata.rs | 73 +- crates/iceberg/src/spec/manifest/_serde.rs | 72 +- crates/iceberg/src/spec/manifest/data_file.rs | 8 +- crates/iceberg/src/spec/manifest/mod.rs | 985 +++++++++++++++++- crates/iceberg/src/spec/manifest/reader.rs | 3 +- crates/iceberg/src/spec/manifest/writer.rs | 108 +- crates/iceberg/src/spec/manifest_list/mod.rs | 6 +- .../iceberg/src/spec/manifest_list/writer.rs | 4 +- crates/iceberg/src/spec/values/serde.rs | 51 +- crates/iceberg/src/spec/values/tests.rs | 70 +- crates/iceberg/testdata/manifests/README.md | 99 ++ .../testdata/manifests/pyiceberg-v2-data.avro | Bin 0 -> 4787 bytes .../repeated-decimal-type-definitions.avro | Bin 0 -> 3865 bytes 20 files changed, 2409 insertions(+), 113 deletions(-) create mode 100644 crates/iceberg/src/avro/deserializer.rs create mode 100644 crates/iceberg/src/avro/named_types.rs create mode 100644 crates/iceberg/testdata/manifests/README.md create mode 100644 crates/iceberg/testdata/manifests/pyiceberg-v2-data.avro create mode 100644 crates/iceberg/testdata/manifests/repeated-decimal-type-definitions.avro diff --git a/Cargo.lock b/Cargo.lock index e2900acb2e..75694c3875 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -155,26 +161,26 @@ checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "apache-avro" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" dependencies = [ "bigdecimal", "bon", "crc32fast", - "digest 0.10.7", + "digest 0.11.3", "log", - "miniz_oxide", + "miniz_oxide 0.9.1", "num-bigint 0.4.8", + "ouroboros", "quad-rand", - "rand 0.9.5", + "rand 0.10.2", "regex-lite", "serde", "serde_bytes", "serde_json", "snap", - "strum", - "strum_macros", + "strum 0.28.0", "thiserror 2.0.18", "uuid", "zstd", @@ -1368,7 +1374,7 @@ version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn", @@ -2105,7 +2111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.9", "zlib-rs", ] @@ -2443,6 +2449,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -2747,10 +2759,11 @@ dependencies = [ "serde_json", "serde_repr", "serde_with", - "strum", + "strum 0.27.2", "tempfile", "tokio", "tracing", + "tracing-subscriber", "typed-builder", "typetag", "url", @@ -2878,7 +2891,7 @@ dependencies = [ "itertools 0.13.0", "regex", "sqlx", - "strum", + "strum 0.27.2", "tempfile", "tokio", "tracing", @@ -3574,6 +3587,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.2.1" @@ -4178,6 +4200,30 @@ dependencies = [ "memchr", ] +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn", +] + [[package]] name = "outref" version = "0.5.2" @@ -4525,6 +4571,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "version_check", + "yansi", +] + [[package]] name = "ptr_meta" version = "0.3.1" @@ -4605,7 +4664,7 @@ version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e343bcec300ff262f5806a33a4e51b6d097a8a46435f512fcb83e95592581625" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn", @@ -5913,7 +5972,7 @@ checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" dependencies = [ "dotenvy", "either", - "heck", + "heck 0.5.0", "hex", "once_cell", "proc-macro2", @@ -6075,7 +6134,16 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", ] [[package]] @@ -6084,7 +6152,19 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ - "heck", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", "proc-macro2", "quote", "syn", diff --git a/Cargo.toml b/Cargo.toml index 661c56b094..24f8819949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ unused_qualifications = "deny" [workspace.dependencies] aes-gcm = "0.10" anyhow = "1.0.72" -apache-avro = { version = "0.21", features = ["snappy", "zstandard"] } +apache-avro = { version = "0.22", features = ["snappy", "zstandard"] } array-init = "2" arrow = "59.2" arrow-arith = "59.2" diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 8783251fd5..f22ed59b9b 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -95,6 +95,7 @@ rand = { workspace = true } regex = { workspace = true } serde_arrow = { version = "0.14", features = ["arrow-59"] } tempfile = { workspace = true } +tracing-subscriber = { workspace = true } [package.metadata.cargo-machete] # These dependencies are added to ensure minimal dependency version diff --git a/crates/iceberg/src/avro/deserializer.rs b/crates/iceberg/src/avro/deserializer.rs new file mode 100644 index 0000000000..e130925443 --- /dev/null +++ b/crates/iceberg/src/avro/deserializer.rs @@ -0,0 +1,457 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Deserialization straight from an Avro writer schema, with the parts of Avro +//! schema resolution that apache-avro's schema-aware deserializer leaves out. +//! +//! `Reader::into_deser_iter` decodes into serde types without building +//! `apache_avro::types::Value`s, but it requires the reader schema to equal the +//! writer schema and applies no resolution rules. [`ResolvingDeserializer`] +//! wraps it and routes every value through `deserialize_any`, which follows the +//! writer schema. As a result: +//! +//! - Avro record names don't have to match serde type names. +//! - A writer value that isn't a union reads into an `Option`. +//! - A writer union reads into a type that isn't an `Option`. A null value +//! returns an error. +//! - serde's numeric visitors convert between numeric types, so an `int` reads +//! into an `i64` and a `float` into an `f64`. An integer that doesn't fit the +//! target, such as a `long` above `i32::MAX` read into an `i32`, returns an +//! error. Conversions into `f32` or `f64` use `as` and can lose precision. +//! +//! apache-avro plans a `SchemaAwareResolvingDeserializer` that resolves against +//! a reader schema (). Once a +//! release includes it, readers can pass their reader schema to +//! `Reader::builder` and drop this module, provided it doesn't reject writer +//! record names that differ from the reader's. The Avro spec requires record +//! names to match, and the writers this crate reads from don't all agree on +//! them. + +use std::fmt; + +use serde::de::value::{ + BorrowedBytesDeserializer, BorrowedStrDeserializer, BytesDeserializer, EnumAccessDeserializer, + MapAccessDeserializer, SeqAccessDeserializer, +}; +use serde::de::{ + DeserializeSeed, Deserializer, EnumAccess, Error, IntoDeserializer, MapAccess, SeqAccess, + Visitor, +}; +use serde::{Deserialize, forward_to_deserialize_any}; + +/// Deserializes the wrapped type through [`ResolvingDeserializer`], for use +/// with `Reader::into_deser_iter`. +pub(crate) struct Resolved(pub T); + +impl<'de, T: Deserialize<'de>> Deserialize<'de> for Resolved { + fn deserialize>(deserializer: D) -> Result { + T::deserialize(ResolvingDeserializer(deserializer)).map(Resolved) + } +} + +struct ResolvingDeserializer(D); + +impl<'de, D: Deserializer<'de>> Deserializer<'de> for ResolvingDeserializer { + type Error = D::Error; + + fn deserialize_any>(self, visitor: V) -> Result { + self.0.deserialize_any(ResolvingVisitor(visitor)) + } + + fn deserialize_option>(self, visitor: V) -> Result { + self.0.deserialize_any(OptionVisitor(visitor)) + } + + fn deserialize_ignored_any>(self, visitor: V) -> Result { + self.0.deserialize_ignored_any(visitor) + } + + fn is_human_readable(&self) -> bool { + self.0.is_human_readable() + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string bytes byte_buf + unit unit_struct newtype_struct seq tuple tuple_struct map struct enum identifier + } +} + +/// Wraps the sequences and maps a visitor receives, so that their elements are +/// deserialized through [`ResolvingDeserializer`] too. +struct ResolvingVisitor(V); + +/// Forwards `visit_*` methods for values that contain no nested values. +macro_rules! forward_visit { + ($($method:ident($ty:ty)),* $(,)?) => { + $( + fn $method(self, v: $ty) -> Result { + self.0.$method(v) + } + )* + }; +} + +impl<'de, V: Visitor<'de>> Visitor<'de> for ResolvingVisitor { + type Value = V::Value; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.0.expecting(formatter) + } + + forward_visit! { + visit_bool(bool), visit_i8(i8), visit_i16(i16), visit_i32(i32), visit_i64(i64), + visit_i128(i128), visit_u8(u8), visit_u16(u16), visit_u32(u32), visit_u64(u64), + visit_u128(u128), visit_f32(f32), visit_f64(f64), visit_char(char), visit_str(&str), + visit_borrowed_str(&'de str), visit_string(String), visit_bytes(&[u8]), + visit_borrowed_bytes(&'de [u8]), visit_byte_buf(Vec), + } + + fn visit_none(self) -> Result { + self.0.visit_none() + } + + fn visit_unit(self) -> Result { + self.0.visit_unit() + } + + fn visit_some>(self, deserializer: D) -> Result { + self.0.visit_some(ResolvingDeserializer(deserializer)) + } + + fn visit_newtype_struct>( + self, + deserializer: D, + ) -> Result { + self.0 + .visit_newtype_struct(ResolvingDeserializer(deserializer)) + } + + fn visit_seq>(self, seq: A) -> Result { + self.0.visit_seq(ResolvingAccess(seq)) + } + + fn visit_map>(self, map: A) -> Result { + self.0.visit_map(ResolvingAccess(map)) + } + + fn visit_enum>(self, data: A) -> Result { + self.0.visit_enum(data) + } +} + +/// Reads an `Option` from any writer value. A null becomes `None`, and anything +/// else becomes `Some`, whether or not the writer schema is a union. +struct OptionVisitor(V); + +/// Implements `visit_*` methods that hand a value to `visit_some` through the +/// deserializer that `deserializer` builds from it. +macro_rules! visit_some { + ($($method:ident($ty:ty) => $deserializer:expr),* $(,)?) => { + $( + fn $method(self, v: $ty) -> Result { + self.0.visit_some($deserializer(v)) + } + )* + }; +} + +impl<'de, V: Visitor<'de>> Visitor<'de> for OptionVisitor { + type Value = V::Value; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.0.expecting(formatter) + } + + visit_some! { + visit_bool(bool) => IntoDeserializer::into_deserializer, + visit_i8(i8) => IntoDeserializer::into_deserializer, + visit_i16(i16) => IntoDeserializer::into_deserializer, + visit_i32(i32) => IntoDeserializer::into_deserializer, + visit_i64(i64) => IntoDeserializer::into_deserializer, + visit_i128(i128) => IntoDeserializer::into_deserializer, + visit_u8(u8) => IntoDeserializer::into_deserializer, + visit_u16(u16) => IntoDeserializer::into_deserializer, + visit_u32(u32) => IntoDeserializer::into_deserializer, + visit_u64(u64) => IntoDeserializer::into_deserializer, + visit_u128(u128) => IntoDeserializer::into_deserializer, + visit_f32(f32) => IntoDeserializer::into_deserializer, + visit_f64(f64) => IntoDeserializer::into_deserializer, + visit_char(char) => IntoDeserializer::into_deserializer, + visit_str(&str) => IntoDeserializer::into_deserializer, + visit_borrowed_str(&'de str) => BorrowedStrDeserializer::new, + visit_string(String) => IntoDeserializer::into_deserializer, + visit_bytes(&[u8]) => BytesDeserializer::new, + visit_borrowed_bytes(&'de [u8]) => BorrowedBytesDeserializer::new, + } + + fn visit_byte_buf(self, v: Vec) -> Result { + self.0.visit_some(BytesDeserializer::new(&v)) + } + + fn visit_none(self) -> Result { + self.0.visit_none() + } + + fn visit_unit(self) -> Result { + self.0.visit_none() + } + + fn visit_some>(self, deserializer: D) -> Result { + self.0.visit_some(ResolvingDeserializer(deserializer)) + } + + fn visit_seq>(self, seq: A) -> Result { + self.0 + .visit_some(SeqAccessDeserializer::new(ResolvingAccess(seq))) + } + + fn visit_map>(self, map: A) -> Result { + self.0 + .visit_some(MapAccessDeserializer::new(ResolvingAccess(map))) + } + + fn visit_enum>(self, data: A) -> Result { + self.0.visit_some(EnumAccessDeserializer::new(data)) + } +} + +/// Deserializes the elements of a sequence, or the keys and values of a map, +/// through [`ResolvingDeserializer`]. Keys need it because apache-avro's record +/// field names only support `deserialize_identifier` and `deserialize_any`, +/// and a key read as a `String` calls `deserialize_string`. +struct ResolvingAccess(A); + +impl<'de, A: SeqAccess<'de>> SeqAccess<'de> for ResolvingAccess { + type Error = A::Error; + + fn next_element_seed>( + &mut self, + seed: T, + ) -> Result, A::Error> { + self.0.next_element_seed(ResolvingSeed(seed)) + } + + fn size_hint(&self) -> Option { + self.0.size_hint() + } +} + +impl<'de, A: MapAccess<'de>> MapAccess<'de> for ResolvingAccess { + type Error = A::Error; + + fn next_key_seed>( + &mut self, + seed: K, + ) -> Result, A::Error> { + self.0.next_key_seed(ResolvingSeed(seed)) + } + + fn next_value_seed>(&mut self, seed: T) -> Result { + self.0.next_value_seed(ResolvingSeed(seed)) + } + + fn size_hint(&self) -> Option { + self.0.size_hint() + } +} + +struct ResolvingSeed(S); + +impl<'de, S: DeserializeSeed<'de>> DeserializeSeed<'de> for ResolvingSeed { + type Value = S::Value; + + fn deserialize>(self, deserializer: D) -> Result { + self.0.deserialize(ResolvingDeserializer(deserializer)) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::fmt::Debug; + + use apache_avro::types::Value as AvroValue; + use apache_avro::{Reader, Schema as AvroSchema, Writer}; + use serde::de::DeserializeOwned; + use serde_bytes::ByteBuf; + use serde_derive::Deserialize; + + use super::*; + + #[derive(Deserialize, Debug, PartialEq)] + struct Pair { + a: i64, + b: Option, + } + + #[derive(Deserialize, Debug, PartialEq)] + enum Suit { + Hearts, + Spades, + } + + /// Writes `value` with `writer_schema` and reads it back as a `T`. + fn read( + writer_schema: &str, + value: AvroValue, + ) -> apache_avro::AvroResult { + let schema = AvroSchema::parse_str(writer_schema).unwrap(); + let mut writer = Writer::new(&schema, Vec::new()).unwrap(); + writer.append_value(value).unwrap(); + let bs = writer.into_inner().unwrap(); + let mut values = Reader::new(bs.as_slice()) + .unwrap() + .into_deser_iter::>(); + values.next().unwrap().map(|value| value.0) + } + + fn check( + writer_schema: &str, + value: AvroValue, + expected: T, + ) { + assert_eq!( + read::(writer_schema, value).unwrap(), + expected, + "{writer_schema}" + ); + } + + const PAIR: &str = r#"{"type": "record", "name": "writer_name", "fields": [ + {"name": "b", "type": "string"}, + {"name": "extra", "type": "int"}, + {"name": "a", "type": "int"} + ]}"#; + + fn pair_value() -> AvroValue { + AvroValue::Record(vec![ + ("b".to_string(), AvroValue::String("x".to_string())), + ("extra".to_string(), AvroValue::Int(9)), + ("a".to_string(), AvroValue::Int(1)), + ]) + } + + fn pair() -> Pair { + Pair { + a: 1, + b: Some("x".to_string()), + } + } + + const SUIT: &str = r#"{"type": "enum", "name": "suit", "symbols": ["Hearts", "Spades"]}"#; + + #[test] + fn test_resolved_reads_writer_values() { + check(r#""boolean""#, AvroValue::Boolean(true), true); + check(r#""int""#, AvroValue::Int(-3), -3i64); + check(r#""long""#, AvroValue::Long(7), 7i32); + check(r#""float""#, AvroValue::Float(1.5), 1.5f64); + check(r#""double""#, AvroValue::Double(2.5), 2.5f64); + check( + r#""string""#, + AvroValue::String("s".to_string()), + "s".to_string(), + ); + check( + r#""bytes""#, + AvroValue::Bytes(vec![1, 2]), + ByteBuf::from(vec![1, 2]), + ); + check( + r#"{"type": "fixed", "name": "f", "size": 2}"#, + AvroValue::Fixed(2, vec![3, 4]), + ByteBuf::from(vec![3, 4]), + ); + check( + r#"{"type": "array", "items": "int"}"#, + AvroValue::Array(vec![AvroValue::Int(1), AvroValue::Int(2)]), + vec![1i64, 2], + ); + check( + r#"{"type": "map", "values": "long"}"#, + AvroValue::Map(HashMap::from([("k".to_string(), AvroValue::Long(5))])), + HashMap::from([("k".to_string(), 5i32)]), + ); + check(PAIR, pair_value(), pair()); + check(SUIT, AvroValue::Enum(1, "Spades".to_string()), Suit::Spades); + check( + r#"["null", "long"]"#, + AvroValue::Union(1, Box::new(AvroValue::Long(5))), + 5i64, + ); + } + + #[test] + fn test_resolved_reads_any_writer_value_as_option() { + check(r#""null""#, AvroValue::Null, None::); + check( + r#"["null", "long"]"#, + AvroValue::Union(0, Box::new(AvroValue::Null)), + None::, + ); + check( + r#"["null", "int"]"#, + AvroValue::Union(1, Box::new(AvroValue::Int(5))), + Some(5i64), + ); + check(r#""boolean""#, AvroValue::Boolean(true), Some(true)); + check(r#""int""#, AvroValue::Int(-3), Some(-3i64)); + check(r#""long""#, AvroValue::Long(7), Some(7i32)); + check(r#""float""#, AvroValue::Float(1.5), Some(1.5f64)); + check(r#""double""#, AvroValue::Double(2.5), Some(2.5f64)); + check( + r#""string""#, + AvroValue::String("s".to_string()), + Some("s".to_string()), + ); + check( + r#""bytes""#, + AvroValue::Bytes(vec![1, 2]), + Some(ByteBuf::from(vec![1, 2])), + ); + check( + r#"{"type": "array", "items": "int"}"#, + AvroValue::Array(vec![AvroValue::Int(1)]), + Some(vec![1i64]), + ); + check( + r#"{"type": "map", "values": "long"}"#, + AvroValue::Map(HashMap::from([("k".to_string(), AvroValue::Long(5))])), + Some(HashMap::from([("k".to_string(), 5i32)])), + ); + check(PAIR, pair_value(), Some(pair())); + check( + SUIT, + AvroValue::Enum(0, "Hearts".to_string()), + Some(Suit::Hearts), + ); + } + + #[test] + fn test_resolved_rejects_values_that_do_not_fit() { + assert!(read::(r#""long""#, AvroValue::Long(i64::from(i32::MAX) + 1)).is_err()); + assert!(read::(r#""int""#, AvroValue::Int(-1)).is_err()); + assert!( + read::( + r#"["null", "long"]"#, + AvroValue::Union(0, Box::new(AvroValue::Null)) + ) + .is_err() + ); + assert!(read::(r#""long""#, AvroValue::Long(1)).is_err()); + } +} diff --git a/crates/iceberg/src/avro/mod.rs b/crates/iceberg/src/avro/mod.rs index f2a9310e7c..0bc5a70041 100644 --- a/crates/iceberg/src/avro/mod.rs +++ b/crates/iceberg/src/avro/mod.rs @@ -16,5 +16,11 @@ // under the License. //! Avro related codes. +mod deserializer; +mod named_types; mod schema; +pub(crate) use deserializer::Resolved; +pub(crate) use named_types::define_named_types_once; +#[cfg(test)] +pub(crate) use named_types::define_named_types_repeatedly; pub(crate) use schema::*; diff --git a/crates/iceberg/src/avro/named_types.rs b/crates/iceberg/src/avro/named_types.rs new file mode 100644 index 0000000000..a006a84156 --- /dev/null +++ b/crates/iceberg/src/avro/named_types.rs @@ -0,0 +1,309 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reading Avro files whose schema defines a named type more than once. +//! +//! The Avro specification allows one definition per full name. iceberg-rust +//! wrote manifests that define a `fixed` or `decimal` type once for every field +//! that uses it, while `apache-avro` 0.21 accepted that. `apache-avro` 0.22 +//! rejects such a file's header. Rewriting each repeated definition as a +//! reference to the first one keeps those files readable, because a reference +//! decodes the same bytes as the definition it refers to. + +use std::collections::HashMap; +use std::io::Cursor; + +use apache_avro::Schema as AvroSchema; +use apache_avro::reader::datum::GenericDatumReader; +use apache_avro::schema::MapSchema; +use apache_avro::types::Value as AvroValue; +use apache_avro::writer::datum::GenericDatumWriter; +use serde_json::{Map, Value as JsonValue}; + +use crate::Result; +use crate::error::invalid_data; + +const MAGIC: &[u8] = b"Obj\x01"; +const SCHEMA_KEY: &str = "avro.schema"; + +/// Rewrites the header of an Avro object container file so that each named type +/// is defined once and referenced by name afterwards. Returns the rewritten +/// file and the full names of the repeated types, or `None` if a repeated +/// definition differs from the first one, which leaves the file ambiguous. +pub(crate) fn define_named_types_once(bs: &[u8]) -> Result, Vec)>> { + rewrite_schema(bs, reference_repeated_definitions) +} + +/// Rewrites the schema in the header of an Avro object container file with +/// `rewrite`, leaving the data blocks as they are. Returns `None` if `rewrite` +/// does. +fn rewrite_schema( + bs: &[u8], + rewrite: impl FnOnce(&mut JsonValue) -> Option, +) -> Result, T)>> { + let body = bs + .strip_prefix(MAGIC) + .ok_or_else(|| invalid_data!("Not an Avro object container file"))?; + // The header's metadata is an Avro `map`, followed by the sync marker. + let metadata_schema = AvroSchema::Map(MapSchema { + types: Box::new(AvroSchema::Bytes), + attributes: Default::default(), + }); + let mut cursor = Cursor::new(body); + let AvroValue::Map(mut metadata) = GenericDatumReader::builder(&metadata_schema) + .build()? + .read_value(&mut cursor)? + else { + return Err(invalid_data!("Avro file metadata is not a map")); + }; + let rest = usize::try_from(cursor.position()) + .ok() + .and_then(|position| body.get(position..)) + .ok_or_else(|| invalid_data!("Avro file metadata extends past the end of the file"))?; + + let Some(AvroValue::Bytes(schema)) = metadata.get(SCHEMA_KEY) else { + return Err(invalid_data!("Avro file metadata has no {SCHEMA_KEY}")); + }; + let mut schema: JsonValue = serde_json::from_slice(schema)?; + let Some(rewritten) = rewrite(&mut schema) else { + return Ok(None); + }; + metadata.insert( + SCHEMA_KEY.to_string(), + AvroValue::Bytes(serde_json::to_vec(&schema)?), + ); + let metadata = GenericDatumWriter::builder(&metadata_schema) + .build()? + .write_value_to_vec(AvroValue::Map(metadata))?; + + Ok(Some(([MAGIC, &metadata, rest].concat(), rewritten))) +} + +/// Rewrites the header of an Avro object container file so that each reference +/// to a named type repeats its definition, as iceberg-rust wrote manifests with +/// `apache-avro` 0.21. +#[cfg(test)] +pub(crate) fn define_named_types_repeatedly(bs: &[u8]) -> Vec { + fn inline_references(schema: &mut JsonValue, defined: &mut HashMap) { + match schema { + JsonValue::String(name) => { + if let Some(definition) = defined.get(name.as_str()) { + *schema = definition.clone(); + } + } + JsonValue::Array(schemas) => schemas + .iter_mut() + .for_each(|schema| inline_references(schema, defined)), + JsonValue::Object(object) => { + if let (Some(JsonValue::String(name)), Some("record" | "enum" | "fixed")) = ( + object.get("name"), + object.get("type").and_then(JsonValue::as_str), + ) { + defined.insert(name.clone(), JsonValue::Object(object.clone())); + } + for key in ["type", "items", "values", "fields"] { + if let Some(child) = object.get_mut(key) { + inline_references(child, defined); + } + } + } + _ => {} + } + } + + rewrite_schema(bs, |schema| { + inline_references(schema, &mut HashMap::new()); + Some(()) + }) + .unwrap() + .unwrap() + .0 +} + +/// Replaces each repeated definition of a named type in `schema` with its full +/// name. Returns the replaced names, or `None` if a repeated definition differs +/// from the first one. +fn reference_repeated_definitions(schema: &mut JsonValue) -> Option> { + let mut defined = HashMap::new(); + let mut repeated = Vec::new(); + reference_repeated(schema, None, &mut defined, &mut repeated).then_some(repeated) +} + +/// Walks `schema` in the order Avro resolves names, recording each named type's +/// definition without its name. Returns `false` on a repeated name whose +/// definition differs from the first one. +fn reference_repeated( + schema: &mut JsonValue, + namespace: Option<&str>, + defined: &mut HashMap>, + repeated: &mut Vec, +) -> bool { + let object = match schema { + JsonValue::Array(variants) => { + return variants + .iter_mut() + .all(|variant| reference_repeated(variant, namespace, defined, repeated)); + } + JsonValue::Object(object) => object, + _ => return true, + }; + let type_name = match object.get_mut("type") { + Some(JsonValue::String(type_name)) => type_name.as_str(), + Some(nested) => return reference_repeated(nested, namespace, defined, repeated), + None => return true, + }; + match type_name { + "record" | "error" | "enum" | "fixed" => {} + "array" => { + return object + .get_mut("items") + .is_none_or(|items| reference_repeated(items, namespace, defined, repeated)); + } + "map" => { + return object + .get_mut("values") + .is_none_or(|values| reference_repeated(values, namespace, defined, repeated)); + } + _ => return true, + } + + let Some(full_name) = full_name(object, namespace) else { + // Leave a malformed schema for apache-avro to report. + return true; + }; + let mut definition = object.clone(); + definition.remove("name"); + definition.remove("namespace"); + if let Some(first) = defined.get(&full_name) { + if *first != definition { + return false; + } + *schema = JsonValue::String(full_name.clone()); + repeated.push(full_name); + return true; + } + defined.insert(full_name.clone(), definition); + + let record_namespace = full_name.rsplit_once('.').map(|(namespace, _)| namespace); + match object.get_mut("fields") { + Some(JsonValue::Array(fields)) => fields.iter_mut().all(|field| { + field.get_mut("type").is_none_or(|field_type| { + reference_repeated(field_type, record_namespace, defined, repeated) + }) + }), + _ => true, + } +} + +/// The full name of a named type, following the Avro specification: a name +/// containing a dot is already full, and otherwise the type's own `namespace` +/// or the enclosing namespace qualifies it. +fn full_name(object: &Map, enclosing_namespace: Option<&str>) -> Option { + let name = object.get("name")?.as_str()?; + if name.contains('.') { + return Some(name.to_string()); + } + let namespace = match object.get("namespace") { + Some(namespace) => namespace.as_str(), + None => enclosing_namespace, + }; + Some(match namespace { + Some(namespace) if !namespace.is_empty() => format!("{namespace}.{name}"), + _ => name.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn decimal_10_2() -> JsonValue { + json!({"type": "fixed", "name": "decimal_10_2", "size": 5, + "logicalType": "decimal", "precision": 10, "scale": 2}) + } + + #[test] + fn test_reference_repeated_definitions() { + let mut schema = json!({"type": "record", "name": "r102", "fields": [ + {"name": "a", "type": ["null", decimal_10_2()]}, + {"name": "b", "type": ["null", decimal_10_2()]}, + {"name": "c", "type": {"type": "array", "items": decimal_10_2()}}, + ]}); + + let repeated = reference_repeated_definitions(&mut schema); + + assert_eq!( + repeated, + Some(vec!["decimal_10_2".to_string(), "decimal_10_2".to_string()]) + ); + assert_eq!( + schema, + json!({"type": "record", "name": "r102", "fields": [ + {"name": "a", "type": ["null", decimal_10_2()]}, + {"name": "b", "type": ["null", "decimal_10_2"]}, + {"name": "c", "type": {"type": "array", "items": "decimal_10_2"}}, + ]}) + ); + } + + #[test] + fn test_reference_repeated_definitions_leaves_unique_names() { + let original = json!({"type": "record", "name": "r", "fields": [ + {"name": "a", "type": decimal_10_2()}, + {"name": "b", "type": "decimal_10_2"}, + {"name": "c", "type": {"type": "fixed", "name": "fixed_4", "size": 4}}, + ]}); + let mut schema = original.clone(); + + assert_eq!(reference_repeated_definitions(&mut schema), Some(vec![])); + assert_eq!(schema, original); + } + + #[test] + fn test_reference_repeated_definitions_uses_full_names() { + // The same short name in two namespaces names two different types. + let mut schema = json!({"type": "record", "name": "r", "fields": [ + {"name": "a", "type": {"type": "fixed", "name": "f", "namespace": "x", "size": 4}}, + {"name": "b", "type": {"type": "fixed", "name": "f", "namespace": "y", "size": 8}}, + {"name": "c", "type": {"type": "fixed", "name": "x.f", "size": 4}}, + {"name": "d", "type": {"type": "record", "name": "n", "namespace": "x", "fields": [ + {"name": "e", "type": {"type": "fixed", "name": "f", "size": 4}}, + ]}}, + ]}); + + let repeated = reference_repeated_definitions(&mut schema); + + assert_eq!(repeated, Some(vec!["x.f".to_string(), "x.f".to_string()])); + assert_eq!(schema["fields"][2]["type"], json!("x.f")); + assert_eq!( + schema["fields"][3]["type"]["fields"][0]["type"], + json!("x.f") + ); + } + + #[test] + fn test_reference_repeated_definitions_rejects_different_definitions() { + let mut schema = json!({"type": "record", "name": "r", "fields": [ + {"name": "a", "type": {"type": "fixed", "name": "f", "size": 4}}, + {"name": "b", "type": {"type": "fixed", "name": "f", "size": 8}}, + ]}); + + assert_eq!(reference_repeated_definitions(&mut schema), None); + } +} diff --git a/crates/iceberg/src/avro/schema.rs b/crates/iceberg/src/avro/schema.rs index 908dff5c9f..47254317ed 100644 --- a/crates/iceberg/src/avro/schema.rs +++ b/crates/iceberg/src/avro/schema.rs @@ -16,12 +16,12 @@ // under the License. //! Conversion between iceberg and avro schema. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use apache_avro::Schema as AvroSchema; use apache_avro::schema::{ - ArraySchema, DecimalSchema, FixedSchema, MapSchema, Name, RecordField as AvroRecordField, - RecordFieldOrder, RecordSchema, UnionSchema, + ArraySchema, DecimalSchema, FixedSchema, InnerDecimalSchema, MapSchema, Name, + RecordField as AvroRecordField, RecordSchema, UnionSchema, UuidSchema, }; use itertools::{Either, Itertools}; use serde_json::{Number, Value}; @@ -43,6 +43,29 @@ const LOGICAL_TYPE: &str = "logicalType"; struct SchemaToAvroSchema { schema: String, + /// Names of the `fixed` types defined so far. Avro allows one definition per + /// name, and the visitor reaches fields in the order they're serialized. + defined_names: HashSet, +} + +impl SchemaToAvroSchema { + /// Returns a reference to `schema` by name if a type with its name was + /// already defined, and `schema` itself otherwise. + fn define_once(&mut self, schema: AvroSchema) -> AvroSchema { + let name = match &schema { + AvroSchema::Fixed(FixedSchema { name, .. }) + | AvroSchema::Decimal(DecimalSchema { + inner: InnerDecimalSchema::Fixed(FixedSchema { name, .. }), + .. + }) => name.clone(), + _ => return schema, + }; + if self.defined_names.insert(name.clone()) { + schema + } else { + AvroSchema::Ref { name } + } + } } type AvroSchemaOrField = Either; @@ -54,7 +77,7 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut avro_schema = value.unwrap_left(); if let AvroSchema::Record(record) = &mut avro_schema { - record.name = Name::from(self.schema.as_str()); + record.name = Name::new(self.schema.as_str())?; } else { return Err(Error::new( ErrorKind::Unexpected, @@ -72,7 +95,7 @@ impl SchemaVisitor for SchemaToAvroSchema { ) -> Result { let mut field_schema = avro_schema.unwrap_left(); if let AvroSchema::Record(record) = &mut field_schema { - record.name = Name::from(format!("r{}", field.id).as_str()); + record.name = Name::new(format!("r{}", field.id))?; } if !field.required { @@ -96,10 +119,8 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut avro_record_field = AvroRecordField { name: field.name.clone(), schema: field_schema, - order: RecordFieldOrder::Ignore, - position: 0, doc: field.doc.clone(), - aliases: None, + aliases: Vec::new(), default, custom_attributes: Default::default(), }; @@ -130,7 +151,7 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut field_schema = value.unwrap_left(); if let AvroSchema::Record(record) = &mut field_schema { - record.name = Name::from(format!("r{}", list.element_field.id).as_str()); + record.name = Name::new(format!("r{}", list.element_field.id))?; } if !list.element_field.required { @@ -191,11 +212,9 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut field = AvroRecordField { name: map.key_field.name.clone(), doc: None, - aliases: None, + aliases: Vec::new(), default: None, schema: key_field_schema, - order: RecordFieldOrder::Ascending, - position: 0, custom_attributes: Default::default(), }; field.custom_attributes.insert( @@ -209,11 +228,9 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut field = AvroRecordField { name: map.value_field.name.clone(), doc: None, - aliases: None, + aliases: Vec::new(), default: None, schema: value_field_schema, - order: RecordFieldOrder::Ignore, - position: 0, custom_attributes: Default::default(), }; field.custom_attributes.insert( @@ -254,11 +271,13 @@ impl SchemaVisitor for SchemaToAvroSchema { PrimitiveType::TimestampNs => AvroSchema::TimestampNanos, PrimitiveType::TimestamptzNs => AvroSchema::TimestampNanos, PrimitiveType::String => AvroSchema::String, - PrimitiveType::Uuid => AvroSchema::Uuid, - PrimitiveType::Fixed(len) => avro_fixed_schema((*len) as usize)?, + // TODO(#2913): write the spec's `fixed[16]` uuid. The manifest reader + // already accepts both forms. + PrimitiveType::Uuid => AvroSchema::Uuid(UuidSchema::String), + PrimitiveType::Fixed(len) => self.define_once(avro_fixed_schema((*len) as usize)?), PrimitiveType::Binary => AvroSchema::Bytes, PrimitiveType::Decimal { precision, scale } => { - avro_decimal_schema(*precision as usize, *scale as usize)? + self.define_once(avro_decimal_schema(*precision as usize, *scale as usize)?) } }; Ok(Either::Left(avro_schema)) @@ -276,6 +295,7 @@ impl SchemaVisitor for SchemaToAvroSchema { pub(crate) fn schema_to_avro_schema(name: impl ToString, schema: &Schema) -> Result { let mut converter = SchemaToAvroSchema { schema: name.to_string(), + defined_names: HashSet::new(), }; visit_schema(schema, &mut converter).map(Either::unwrap_left) @@ -305,7 +325,6 @@ pub(crate) fn avro_fixed_schema(len: usize) -> Result { doc: None, size: len, attributes: Default::default(), - default: None, })) } @@ -317,16 +336,15 @@ pub(crate) fn avro_decimal_schema(precision: usize, scale: usize) -> Result Type::Primitive(PrimitiveType::Long), AvroSchema::Float => Type::Primitive(PrimitiveType::Float), AvroSchema::Double => Type::Primitive(PrimitiveType::Double), - AvroSchema::Uuid => Type::Primitive(PrimitiveType::Uuid), + AvroSchema::Uuid(_) => Type::Primitive(PrimitiveType::Uuid), AvroSchema::String | AvroSchema::Enum(_) => Type::Primitive(PrimitiveType::String), AvroSchema::Fixed(fixed) => Type::Primitive(PrimitiveType::Fixed(fixed.size as u64)), AvroSchema::Bytes => Type::Primitive(PrimitiveType::Binary), @@ -627,7 +645,7 @@ mod tests { use std::sync::Arc; use apache_avro::Schema as AvroSchema; - use apache_avro::schema::{Namespace, UnionSchema}; + use apache_avro::schema::UnionSchema; use super::*; use crate::avro::schema::AvroSchemaToSchema; @@ -656,11 +674,9 @@ mod tests { assert_eq!(iceberg_schema, converted_iceberg_schema); // 2. iceberg to avro - let converted_avro_schema = schema_to_avro_schema( - avro_schema.name().unwrap().fullname(Namespace::None), - &iceberg_schema, - ) - .unwrap(); + let converted_avro_schema = + schema_to_avro_schema(avro_schema.name().unwrap().fullname(None), &iceberg_schema) + .unwrap(); assert_eq!(avro_schema, converted_avro_schema); // 3.iceberg to avro to iceberg back @@ -975,6 +991,79 @@ mod tests { } #[test] + fn test_schema_to_avro_schema_defines_named_types_before_references() { + let decimal = Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }); + let iceberg_schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "a", decimal.clone()).into(), + NestedField::required( + 2, + "list", + Type::List(ListType::new( + NestedField::list_element(3, decimal.clone(), false).into(), + )), + ) + .into(), + NestedField::required( + 4, + "map", + Type::Map(MapType { + key_field: NestedField::map_key_element(5, PrimitiveType::Int.into()) + .into(), + value_field: NestedField::map_value_element(6, decimal, false).into(), + }), + ) + .into(), + ]) + .build() + .unwrap(); + + let json = schema_to_avro_schema("r", &iceberg_schema) + .unwrap() + .canonical_form(); + + // Parsing fails on a reference that precedes its definition. + AvroSchema::parse_str(&json).unwrap(); + assert_eq!(json.matches(r#""name":"decimal_10_2""#).count(), 1); + } + + #[test] + fn test_schema_with_array_map_without_parsing() { + // Unlike `test_schema_with_array_map`, this converts an Avro schema built + // in code, which keeps `logicalType: map` in apache-avro 0.22. + let map = |key_id, value_id, value_required| { + Type::Map(MapType { + key_field: NestedField::map_key_element(key_id, PrimitiveType::Boolean.into()) + .into(), + value_field: NestedField::map_value_element( + value_id, + PrimitiveType::Boolean.into(), + value_required, + ) + .into(), + }) + }; + let iceberg_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(100, "optional_values", map(102, 103, false)).into(), + NestedField::optional(104, "required_values", map(105, 106, true)).into(), + ]) + .build() + .unwrap(); + + let avro_schema = schema_to_avro_schema("avro_schema", &iceberg_schema).unwrap(); + + assert_eq!(avro_schema_to_schema(&avro_schema).unwrap(), iceberg_schema); + } + + #[test] + #[ignore = "apache-avro 0.22 drops `logicalType: map` when parsing schema JSON \ + (https://github.com/apache/avro-rs/issues/654), so avro_schema_to_schema \ + reads a parsed map array as a list. Fixed by \ + https://github.com/apache/avro-rs/pull/656, which is not yet released."] fn test_schema_with_array_map() { let avro_schema = { AvroSchema::parse_str( @@ -1202,7 +1291,14 @@ mod tests { fn test_unknown_primitive() { let mut converter = AvroSchemaToSchema; - assert!(converter.primitive(&AvroSchema::Duration).is_err()); + let duration = AvroSchema::Duration(FixedSchema { + name: Name::new("duration").unwrap(), + aliases: None, + doc: None, + size: 12, + attributes: Default::default(), + }); + assert!(converter.primitive(&duration).is_err()); } #[test] diff --git a/crates/iceberg/src/encryption/key_metadata.rs b/crates/iceberg/src/encryption/key_metadata.rs index d56c8dbd3b..b3222eec53 100644 --- a/crates/iceberg/src/encryption/key_metadata.rs +++ b/crates/iceberg/src/encryption/key_metadata.rs @@ -98,7 +98,7 @@ impl StandardKeyMetadata { /// Encodes to Java-compatible format: `[0x01] [Avro binary datum]` pub fn encode(&self) -> Result> { - _serde::StandardKeyMetadataV1::from(self).encode() + _serde::StandardKeyMetadataV1::try_from(self)?.encode() } /// Decodes from Java-compatible format. @@ -131,7 +131,9 @@ mod _serde { use std::io::Cursor; use std::sync::{Arc, LazyLock}; - use apache_avro::{Schema as AvroSchema, from_avro_datum, from_value, to_avro_datum, to_value}; + use apache_avro::reader::datum::GenericDatumReader; + use apache_avro::writer::datum::GenericDatumWriter; + use apache_avro::{Schema as AvroSchema, from_value, to_value}; use serde::{Deserialize, Serialize}; use super::*; @@ -173,7 +175,7 @@ mod _serde { pub(super) struct StandardKeyMetadataV1 { pub encryption_key: serde_bytes::ByteBuf, pub aad_prefix: Option, - pub file_length: Option, + pub file_length: Option, } impl StandardKeyMetadataV1 { @@ -185,9 +187,13 @@ mod _serde { .with_source(e) })?; - let datum = to_avro_datum(&AVRO_SCHEMA_V1, value).map_err(|e| { - Error::new(ErrorKind::Unexpected, "Failed to encode key metadata").with_source(e) - })?; + let datum = GenericDatumWriter::builder(&AVRO_SCHEMA_V1) + .build() + .and_then(|writer| writer.write_value_to_vec(value)) + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "Failed to encode key metadata") + .with_source(e) + })?; let mut result = Vec::with_capacity(1 + datum.len()); result.push(V1); @@ -209,7 +215,9 @@ mod _serde { } let mut reader = Cursor::new(&bytes[1..]); - let value = from_avro_datum(&AVRO_SCHEMA_V1, &mut reader, None) + let value = GenericDatumReader::builder(&AVRO_SCHEMA_V1) + .build() + .and_then(|datum_reader| datum_reader.read_value(&mut reader)) .map_err(|e| invalid_data!("Failed to decode key metadata").with_source(e))?; from_value(&value) @@ -217,16 +225,29 @@ mod _serde { } } - impl From<&StandardKeyMetadata> for StandardKeyMetadataV1 { - fn from(metadata: &StandardKeyMetadata) -> Self { - Self { + impl TryFrom<&StandardKeyMetadata> for StandardKeyMetadataV1 { + type Error = Error; + + fn try_from(metadata: &StandardKeyMetadata) -> Result { + let file_length = metadata + .file_length + .map(i64::try_from) + .transpose() + .map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + "Key metadata file length exceeds the Avro long range", + ) + .with_source(e) + })?; + Ok(Self { encryption_key: serde_bytes::ByteBuf::from(metadata.encryption_key.as_bytes()), aad_prefix: metadata .aad_prefix .as_ref() .map(|b| serde_bytes::ByteBuf::from(b.as_ref())), - file_length: metadata.file_length, - } + file_length, + }) } } @@ -240,7 +261,9 @@ mod _serde { Ok(Self { encryption_key, aad_prefix: v1.aad_prefix.map(|b| b.into_vec().into_boxed_slice()), - file_length: v1.file_length, + file_length: v1.file_length.map(u64::try_from).transpose().map_err(|e| { + invalid_data!("Negative file length in key metadata").with_source(e) + })?, }) } } @@ -328,6 +351,30 @@ mod tests { } } + #[test] + fn test_encode_rejects_file_length_above_long_max() { + let metadata = StandardKeyMetadata::try_new(&[0u8; 16]) + .unwrap() + .with_file_length(i64::MAX as u64 + 1); + + let err = metadata.encode().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_decode_rejects_negative_file_length() { + let serialized = _serde::StandardKeyMetadataV1 { + encryption_key: serde_bytes::ByteBuf::from(vec![0u8; 16]), + aad_prefix: None, + file_length: Some(-1), + } + .encode() + .unwrap(); + + let err = StandardKeyMetadata::decode(&serialized).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + #[test] fn test_decode_rejects_invalid_key_length() { // Craft wire bytes carrying an invalid-length DEK directly via the diff --git a/crates/iceberg/src/spec/manifest/_serde.rs b/crates/iceberg/src/spec/manifest/_serde.rs index 336c883f1a..7d89af64a9 100644 --- a/crates/iceberg/src/spec/manifest/_serde.rs +++ b/crates/iceberg/src/spec/manifest/_serde.rs @@ -15,6 +15,15 @@ // specific language governing permissions and limitations // under the License. +//! Serde forms of manifest entries. +//! +//! `Manifest::parse_avro` deserializes these structs from the writer schema +//! without a reader schema, so they decide which manifests read. A field that +//! a writer may omit, per the spec's read rules for every format version the +//! struct reads, must be an `Option` or have `#[serde(default)]`. +//! `test_parse_manifest_without_each_field` and +//! `test_parse_v1_manifest_without_each_field` check each field. + use std::collections::HashMap; use serde_derive::{Deserialize, Serialize}; @@ -58,6 +67,7 @@ impl ManifestEntryV2 { file_sequence_number: self.file_sequence_number, data_file: self .data_file + .project_partition_by_name(partition_type)? .try_into(partition_spec_id, partition_type, schema)?, }) } @@ -66,6 +76,8 @@ impl ManifestEntryV2 { #[derive(Serialize, Deserialize)] pub(super) struct ManifestEntryV1 { status: i32, + // Required in v1 by the spec. Java declares it optional and writes null when + // snapshot ID inheritance is enabled, which #3371 tracks. pub snapshot_id: i64, data_file: DataFileSerde, } @@ -92,6 +104,7 @@ impl ManifestEntryV1 { file_sequence_number: Some(0), data_file: self .data_file + .project_partition_by_name(partition_type)? .try_into(partition_spec_id, partition_type, schema)?, }) } @@ -161,6 +174,16 @@ impl DataFileSerde { }) } + /// Matches the partition value's fields to `partition_type` by name. Manifest + /// entries are deserialized with the writer's partition fields and order, + /// which can differ from the partition spec's. + fn project_partition_by_name(mut self, partition_type: &Type) -> Result { + if let Type::Struct(struct_type) = partition_type { + self.partition = self.partition.project_by_name(struct_type)?; + } + Ok(self) + } + pub fn try_into( self, partition_spec_id: i32, @@ -315,9 +338,10 @@ mod tests { use std::io::Cursor; use std::sync::Arc; + use crate::avro::define_named_types_repeatedly; use crate::spec::manifest::_serde::{I64Entry, parse_i64_entry}; use crate::spec::{ - DataContentType, DataFile, DataFileFormat, Datum, FormatVersion, NestedField, + DataContentType, DataFile, DataFileFormat, Datum, FormatVersion, Literal, NestedField, PrimitiveType, Schema, Struct, StructType, Type, read_data_files_from_avro, write_data_files_to_avro, }; @@ -412,6 +436,52 @@ mod tests { assert_eq!(data_files, actual_data_file); } + /// apache-avro 0.22 checks the writer schema for repeated names only when + /// the reader has no reader schema, so this path reads 0.21 output without + /// the header rewrite that `Manifest::parse_avro` needs. + #[test] + fn test_read_data_files_with_repeated_named_type_definitions() { + let decimal = Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }); + let schema = Schema::builder() + .with_fields(schema().as_struct().fields().to_vec()) + .with_fields(vec![ + NestedField::optional(4, "a", decimal.clone()).into(), + NestedField::optional(5, "b", decimal.clone()).into(), + ]) + .build() + .unwrap(); + let partition_type = StructType::new(vec![ + NestedField::optional(1000, "a", decimal.clone()).into(), + NestedField::optional(1001, "b", decimal).into(), + ]); + let mut data_files = data_files(); + data_files[0].partition = + Struct::from_iter([Some(Literal::decimal(12345)), Some(Literal::decimal(-678))]); + let mut buffer = Vec::new(); + write_data_files_to_avro( + &mut buffer, + data_files.clone(), + &partition_type, + FormatVersion::V2, + ) + .unwrap(); + let legacy = define_named_types_repeatedly(&buffer); + + let actual = read_data_files_from_avro( + &mut Cursor::new(legacy), + &schema, + 0, + &partition_type, + FormatVersion::V2, + ) + .unwrap(); + + assert_eq!(actual, data_files); + } + #[tokio::test] async fn test_data_file_serialize_deserialize_v1_data_on_v2_reader() { let schema = schema(); diff --git a/crates/iceberg/src/spec/manifest/data_file.rs b/crates/iceberg/src/spec/manifest/data_file.rs index 1fcdaa3a80..94568ab44e 100644 --- a/crates/iceberg/src/spec/manifest/data_file.rs +++ b/crates/iceberg/src/spec/manifest/data_file.rs @@ -301,7 +301,7 @@ pub fn write_data_files_to_avro( FormatVersion::V2 => data_file_schema_v2(partition_type).unwrap(), FormatVersion::V3 => data_file_schema_v3(partition_type).unwrap(), }; - let mut writer = AvroWriter::new(&avro_schema, writer); + let mut writer = AvroWriter::new(&avro_schema, writer)?; // Wrap once and reuse across files so the field-name lookup is not rebuilt // for each data file. @@ -313,7 +313,7 @@ pub fn write_data_files_to_avro( FormatVersion::V1, )?)? .resolve(&avro_schema)?; - writer.append(value)?; + writer.append_value(value)?; } Ok(writer.flush()?) @@ -333,7 +333,9 @@ pub fn read_data_files_from_avro( FormatVersion::V3 => data_file_schema_v3(partition_type).unwrap(), }; - let reader = AvroReader::with_schema(&avro_schema, reader)?; + let reader = AvroReader::builder(reader) + .reader_schema(&avro_schema) + .build()?; // Wrap once and reuse across files so the field-name lookup is not rebuilt // for each data file. let partition_struct_type = Type::Struct(partition_type.clone()); diff --git a/crates/iceberg/src/spec/manifest/mod.rs b/crates/iceberg/src/spec/manifest/mod.rs index 3bd4dc8ad8..f4269638a2 100644 --- a/crates/iceberg/src/spec/manifest/mod.rs +++ b/crates/iceberg/src/spec/manifest/mod.rs @@ -27,15 +27,22 @@ mod reader; pub use reader::*; mod writer; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; -use apache_avro::{Reader as AvroReader, from_value}; +use apache_avro::Reader as AvroReader; +use apache_avro::error::Details; pub use writer::*; use super::{ Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType, Schema, Struct, Type, UNASSIGNED_SEQUENCE_NUMBER, }; -use crate::error::{Result, invalid_data}; +use crate::avro::{Resolved, define_named_types_once}; +use crate::error::{Error, Result, invalid_data}; + +/// Whether a manifest with repeated Avro named type definitions was logged at +/// warn level. A table written by an affected release can have many of them. +static WARNED_REPEATED_DEFINITIONS: AtomicBool = AtomicBool::new(false); /// A manifest contains metadata and a list of entries. #[derive(Debug, PartialEq, Eq, Clone)] @@ -45,9 +52,53 @@ pub struct Manifest { } impl Manifest { - /// Parse manifest metadata and entries from bytes of avro file. - pub(crate) fn try_from_avro_bytes(bs: &[u8]) -> Result<(ManifestMetadata, Vec)> { - let reader = AvroReader::new(bs)?; + /// Parse manifest metadata and entries from bytes of avro file. `location` + /// names the manifest in warnings. + pub(crate) fn try_from_avro_bytes( + bs: &[u8], + location: Option<&str>, + ) -> Result<(ManifestMetadata, Vec)> { + let rewritten; + let reader = match AvroReader::new(bs) { + Ok(reader) => reader, + // iceberg-rust repeated `decimal` definitions before + // `schema_to_avro_schema` defined each named type once, so this + // fallback stays while tables can contain manifests it wrote. + Err(e) if matches!(e.details(), Details::AmbiguousSchemaDefinition(_)) => { + let (bs, repeated) = match define_named_types_once(bs) { + Ok(Some(rewrite)) => rewrite, + Ok(None) => return Err(e.into()), + Err(rewrite_error) => { + return Err(Error::from(e).with_context( + "error defining each named type once", + rewrite_error.to_string(), + )); + } + }; + rewritten = bs; + let reader = AvroReader::new(rewritten.as_slice()).map_err(|retry_error| { + Error::from(e).with_context( + "error after defining each named type once", + retry_error.to_string(), + ) + })?; + let location = location.unwrap_or(""); + if WARNED_REPEATED_DEFINITIONS.swap(true, Ordering::Relaxed) { + tracing::debug!( + "Manifest {location} defines Avro named types {repeated:?} more than once." + ); + } else { + tracing::warn!( + "Manifest {location} defines Avro named types {repeated:?} more than once, \ + which the Avro specification doesn't allow. Reading it with each repeated \ + definition replaced by a reference to the first. Later manifests like \ + this are logged at debug level." + ); + } + reader + } + Err(e) => return Err(e.into()), + }; // Parse manifest metadata let meta = reader.user_metadata(); @@ -61,35 +112,27 @@ impl Manifest { let partition_struct_type = Type::Struct(partition_type.clone()); let entries = match metadata.format_version { - FormatVersion::V1 => { - let schema = manifest_schema_v1(&partition_type)?; - let reader = AvroReader::with_schema(&schema, bs)?; - reader - .into_iter() - .map(|value| { - from_value::<_serde::ManifestEntryV1>(&value?)?.try_into( - metadata.partition_spec.spec_id(), - &partition_struct_type, - &metadata.schema, - ) - }) - .collect::>>()? - } + FormatVersion::V1 => reader + .into_deser_iter::>() + .map(|entry| { + entry?.0.try_into( + metadata.partition_spec.spec_id(), + &partition_struct_type, + &metadata.schema, + ) + }) + .collect::>>()?, // Manifest Schema & Manifest Entry did not change between V2 and V3 - FormatVersion::V2 | FormatVersion::V3 => { - let schema = manifest_schema_v2(&partition_type)?; - let reader = AvroReader::with_schema(&schema, bs)?; - reader - .into_iter() - .map(|value| { - from_value::<_serde::ManifestEntryV2>(&value?)?.try_into( - metadata.partition_spec.spec_id(), - &partition_struct_type, - &metadata.schema, - ) - }) - .collect::>>()? - } + FormatVersion::V2 | FormatVersion::V3 => reader + .into_deser_iter::>() + .map(|entry| { + entry?.0.try_into( + metadata.partition_spec.spec_id(), + &partition_struct_type, + &metadata.schema, + ) + }) + .collect::>>()?, }; Ok((metadata, entries)) @@ -97,7 +140,7 @@ impl Manifest { /// Parse manifest from bytes of avro file. pub fn parse_avro(bs: &[u8]) -> Result { - let (metadata, entries) = Self::try_from_avro_bytes(bs)?; + let (metadata, entries) = Self::try_from_avro_bytes(bs, None)?; Ok(Self::new(metadata, entries)) } @@ -158,11 +201,15 @@ mod tests { use std::fs; use std::sync::Arc; + use apache_avro::types::Value as AvroValue; use apache_avro::{Codec, Writer, to_value}; use serde_json::{Value, to_vec}; use tempfile::TempDir; + use tracing::Level; + use tracing::subscriber::with_default; use super::*; + use crate::ErrorKind; use crate::io::FileIO; use crate::spec::{Literal, NestedField, PrimitiveType, Struct, Transform, Type}; @@ -356,7 +403,7 @@ mod tests { .partition_type(&metadata.schema) .unwrap(); let avro_schema = manifest_schema_v2(&partition_type).unwrap(); - let mut writer = Writer::with_codec(&avro_schema, Vec::new(), Codec::Snappy); + let mut writer = Writer::with_codec(&avro_schema, Vec::new(), Codec::Snappy).unwrap(); writer .add_user_metadata("schema".to_string(), to_vec(&metadata.schema).unwrap()) .unwrap(); @@ -397,7 +444,7 @@ mod tests { .unwrap() .resolve(&avro_schema) .unwrap(); - writer.append(value).unwrap(); + writer.append_value(value).unwrap(); let bs = writer.into_inner().unwrap(); let parsed_manifest = Manifest::parse_avro(&bs).unwrap(); @@ -1361,4 +1408,870 @@ mod tests { assert_eq!(deserialized_data_file1, original_data_file1); assert_eq!(deserialized_data_file2, original_data_file2); } + + /// Metadata for the writer schema tests: a V2 data manifest whose spec has + /// identity partitions on a long, a string, and a double column. + fn writer_schema_test_metadata() -> ManifestMetadata { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "score", Type::Primitive(PrimitiveType::Double)) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .add_partition_field("name", "name", Transform::Identity) + .unwrap() + .add_partition_field("score", "score", Transform::Identity) + .unwrap() + .build() + .unwrap(); + ManifestMetadata { + schema_id: 0, + schema, + partition_spec, + content: ManifestContentType::Data, + format_version: FormatVersion::V2, + } + } + + /// A V2 manifest entry writer schema with Java's record names and only the + /// required `data_file` fields. + fn v2_writer_schema(partition_fields: Value) -> Value { + serde_json::json!({ + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int", "field-id": 0}, + {"name": "snapshot_id", "type": ["null", "long"], "default": null, "field-id": 1}, + {"name": "sequence_number", "type": ["null", "long"], "default": null, "field-id": 3}, + {"name": "file_sequence_number", "type": ["null", "long"], "default": null, "field-id": 4}, + {"name": "data_file", "field-id": 2, "type": { + "type": "record", + "name": "r2", + "fields": [ + {"name": "content", "type": "int", "field-id": 134}, + {"name": "file_path", "type": "string", "field-id": 100}, + {"name": "file_format", "type": "string", "field-id": 101}, + {"name": "partition", "field-id": 102, "type": { + "type": "record", + "name": "r102", + "fields": partition_fields, + }}, + {"name": "record_count", "type": "long", "field-id": 103}, + {"name": "file_size_in_bytes", "type": "long", "field-id": 104}, + ], + }}, + ], + }) + } + + /// The field named by `path` in a record schema, descending through nested + /// record types. + fn writer_schema_field<'a>(record: &'a mut Value, path: &[&str]) -> &'a mut Value { + let (name, rest) = path.split_first().unwrap(); + let field = record["fields"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|field| field["name"] == *name) + .unwrap(); + if rest.is_empty() { + field + } else { + writer_schema_field(&mut field["type"], rest) + } + } + + fn v2_entry(partition: Value) -> Value { + serde_json::json!({ + "status": 1, + "snapshot_id": 7, + "sequence_number": null, + "file_sequence_number": null, + "data_file": { + "content": 0, + "file_path": "s3://bucket/table/data/a.parquet", + "file_format": "PARQUET", + "partition": partition, + "record_count": 10, + "file_size_in_bytes": 100, + }, + }) + } + + fn expected_entry(partition: Struct) -> ManifestEntry { + ManifestEntry { + status: ManifestStatus::Added, + snapshot_id: Some(7), + sequence_number: None, + file_sequence_number: None, + data_file: DataFile { + content: DataContentType::Data, + file_path: "s3://bucket/table/data/a.parquet".to_string(), + file_format: DataFileFormat::Parquet, + partition, + record_count: 10, + file_size_in_bytes: 100, + column_sizes: HashMap::new(), + value_counts: HashMap::new(), + null_value_counts: HashMap::new(), + nan_value_counts: HashMap::new(), + lower_bounds: HashMap::new(), + upper_bounds: HashMap::new(), + key_metadata: None, + split_offsets: None, + equality_ids: None, + sort_order_id: None, + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + }, + } + } + + /// Writes `entries` as a manifest with the given writer schema, encoding each + /// JSON value with the writer schema's types. + fn write_with_writer_schema( + metadata: &ManifestMetadata, + writer_schema: &Value, + entries: Vec, + ) -> Vec { + write_avro_values_with_writer_schema( + metadata, + writer_schema, + entries + .into_iter() + .map(|entry| AvroValue::try_from(entry).unwrap()) + .collect(), + ) + } + + /// Like [`write_with_writer_schema`], for entries that JSON can't express. + fn write_avro_values_with_writer_schema( + metadata: &ManifestMetadata, + writer_schema: &Value, + entries: Vec, + ) -> Vec { + let avro_schema = apache_avro::Schema::parse(writer_schema).unwrap(); + let mut writer = Writer::new(&avro_schema, Vec::new()).unwrap(); + for (key, value) in [ + ("schema", to_vec(&metadata.schema).unwrap()), + ("schema-id", metadata.schema_id.to_string().into_bytes()), + ( + "partition-spec", + to_vec(&metadata.partition_spec.fields()).unwrap(), + ), + ( + "partition-spec-id", + metadata.partition_spec.spec_id().to_string().into_bytes(), + ), + ( + "format-version", + (metadata.format_version as u8).to_string().into_bytes(), + ), + ("content", metadata.content.to_string().into_bytes()), + ] { + writer.add_user_metadata(key.to_string(), value).unwrap(); + } + for entry in entries { + writer + .append_value(entry.resolve(&avro_schema).unwrap()) + .unwrap(); + } + writer.into_inner().unwrap() + } + + /// Removes the field named by `path` from a record schema and from a record + /// value of that schema. + fn remove_writer_field(schema: &mut Value, entry: &mut AvroValue, path: &[&str]) { + let (name, rest) = path.split_first().unwrap(); + let AvroValue::Record(values) = entry else { + unreachable!("the entry is a record"); + }; + let fields = schema["fields"].as_array_mut().unwrap(); + if rest.is_empty() { + fields.retain(|field| field["name"] != *name); + values.retain(|(value_name, _)| value_name != name); + } else { + let field = fields.iter_mut().find(|field| field["name"] == *name); + let value = values.iter_mut().find(|(value_name, _)| value_name == name); + remove_writer_field(&mut field.unwrap()["type"], &mut value.unwrap().1, rest); + } + } + + type ResetField = fn(&mut ManifestEntry); + + /// Removes each field in `optional` and `required` in turn from a manifest + /// that `full_schema` and `full_value` write as `full_entry`. Checks that the + /// manifest reads with an optional field reset and fails naming a required one. + fn assert_reads_without_each_field( + metadata: &ManifestMetadata, + full_schema: &Value, + full_value: &AvroValue, + full_entry: &ManifestEntry, + optional: &[(&[&str], ResetField)], + required: &[&[&str]], + ) { + for (path, reset) in optional { + let (mut schema, mut value) = (full_schema.clone(), full_value.clone()); + remove_writer_field(&mut schema, &mut value, path); + let bs = write_avro_values_with_writer_schema(metadata, &schema, vec![value]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let mut expected = full_entry.clone(); + reset(&mut expected); + assert_eq!( + manifest, + Manifest::new(metadata.clone(), vec![expected]), + "{path:?}" + ); + } + + for path in required { + let (mut schema, mut value) = (full_schema.clone(), full_value.clone()); + remove_writer_field(&mut schema, &mut value, path); + let bs = write_avro_values_with_writer_schema(metadata, &schema, vec![value]); + + let err = Manifest::parse_avro(&bs).unwrap_err(); + + assert!( + err.to_string().contains(path.last().unwrap()), + "{path:?}: {err}" + ); + } + } + + #[test] + fn test_parse_manifest_without_each_field() { + // Fields that the spec doesn't require in every version must read as + // their default when the writer omits them. + let metadata = writer_schema_test_metadata(); + let partition_type = metadata + .partition_spec + .partition_type(&metadata.schema) + .unwrap(); + let full_schema = + serde_json::to_value(manifest_schema_v2(&partition_type).unwrap()).unwrap(); + let mut full_entry = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + full_entry.sequence_number = Some(3); + full_entry.file_sequence_number = Some(4); + let data_file = &mut full_entry.data_file; + data_file.content = DataContentType::PositionDeletes; + data_file.column_sizes = HashMap::from([(1, 40)]); + data_file.value_counts = HashMap::from([(1, 10)]); + data_file.null_value_counts = HashMap::from([(1, 1)]); + data_file.nan_value_counts = HashMap::from([(3, 2)]); + data_file.lower_bounds = HashMap::from([(1, Datum::long(1))]); + data_file.upper_bounds = HashMap::from([(1, Datum::long(9))]); + data_file.key_metadata = Some(vec![1, 2]); + data_file.split_offsets = Some(vec![4]); + data_file.equality_ids = Some(vec![1]); + data_file.sort_order_id = Some(0); + data_file.first_row_id = Some(100); + data_file.referenced_data_file = Some("s3://bucket/table/data/b.parquet".to_string()); + data_file.content_offset = Some(4); + data_file.content_size_in_bytes = Some(8); + let full_value = to_value( + _serde::ManifestEntryV2::try_from( + full_entry.clone(), + &Type::Struct(partition_type.clone()), + ) + .unwrap(), + ) + .unwrap(); + + let optional: [(&[&str], ResetField); 18] = [ + (&["snapshot_id"], |e| e.snapshot_id = None), + (&["sequence_number"], |e| e.sequence_number = None), + (&["file_sequence_number"], |e| e.file_sequence_number = None), + (&["data_file", "content"], |e| { + e.data_file.content = DataContentType::Data + }), + (&["data_file", "column_sizes"], |e| { + e.data_file.column_sizes.clear() + }), + (&["data_file", "value_counts"], |e| { + e.data_file.value_counts.clear() + }), + (&["data_file", "null_value_counts"], |e| { + e.data_file.null_value_counts.clear() + }), + (&["data_file", "nan_value_counts"], |e| { + e.data_file.nan_value_counts.clear() + }), + (&["data_file", "lower_bounds"], |e| { + e.data_file.lower_bounds.clear() + }), + (&["data_file", "upper_bounds"], |e| { + e.data_file.upper_bounds.clear() + }), + (&["data_file", "key_metadata"], |e| { + e.data_file.key_metadata = None + }), + (&["data_file", "split_offsets"], |e| { + e.data_file.split_offsets = None + }), + (&["data_file", "equality_ids"], |e| { + e.data_file.equality_ids = None + }), + (&["data_file", "sort_order_id"], |e| { + e.data_file.sort_order_id = None + }), + (&["data_file", "first_row_id"], |e| { + e.data_file.first_row_id = None + }), + (&["data_file", "referenced_data_file"], |e| { + e.data_file.referenced_data_file = None + }), + (&["data_file", "content_offset"], |e| { + e.data_file.content_offset = None + }), + (&["data_file", "content_size_in_bytes"], |e| { + e.data_file.content_size_in_bytes = None + }), + ]; + let required: [&[&str]; 7] = [ + &["status"], + &["data_file"], + &["data_file", "file_path"], + &["data_file", "file_format"], + &["data_file", "partition"], + &["data_file", "record_count"], + &["data_file", "file_size_in_bytes"], + ]; + assert_reads_without_each_field( + &metadata, + &full_schema, + &full_value, + &full_entry, + &optional, + &required, + ); + } + + #[test] + fn test_parse_v1_manifest_without_each_field() { + let mut metadata = writer_schema_test_metadata(); + metadata.format_version = FormatVersion::V1; + let partition_type = metadata + .partition_spec + .partition_type(&metadata.schema) + .unwrap(); + let full_schema = + serde_json::to_value(manifest_schema_v1(&partition_type).unwrap()).unwrap(); + let mut full_entry = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + // V1 has no sequence numbers, and every V1 entry reads with 0. + full_entry.sequence_number = Some(0); + full_entry.file_sequence_number = Some(0); + let data_file = &mut full_entry.data_file; + data_file.column_sizes = HashMap::from([(1, 40)]); + data_file.value_counts = HashMap::from([(1, 10)]); + data_file.null_value_counts = HashMap::from([(1, 1)]); + data_file.nan_value_counts = HashMap::from([(3, 2)]); + data_file.lower_bounds = HashMap::from([(1, Datum::long(1))]); + data_file.upper_bounds = HashMap::from([(1, Datum::long(9))]); + data_file.key_metadata = Some(vec![1, 2]); + data_file.split_offsets = Some(vec![4]); + data_file.sort_order_id = Some(0); + let full_value = to_value( + _serde::ManifestEntryV1::try_from( + full_entry.clone(), + &Type::Struct(partition_type.clone()), + ) + .unwrap(), + ) + .unwrap(); + + let optional: [(&[&str], ResetField); 10] = [ + // Required in V1, but deprecated and not read. + (&["data_file", "block_size_in_bytes"], |_| {}), + (&["data_file", "column_sizes"], |e| { + e.data_file.column_sizes.clear() + }), + (&["data_file", "value_counts"], |e| { + e.data_file.value_counts.clear() + }), + (&["data_file", "null_value_counts"], |e| { + e.data_file.null_value_counts.clear() + }), + (&["data_file", "nan_value_counts"], |e| { + e.data_file.nan_value_counts.clear() + }), + (&["data_file", "lower_bounds"], |e| { + e.data_file.lower_bounds.clear() + }), + (&["data_file", "upper_bounds"], |e| { + e.data_file.upper_bounds.clear() + }), + (&["data_file", "key_metadata"], |e| { + e.data_file.key_metadata = None + }), + (&["data_file", "split_offsets"], |e| { + e.data_file.split_offsets = None + }), + (&["data_file", "sort_order_id"], |e| { + e.data_file.sort_order_id = None + }), + ]; + let required: [&[&str]; 8] = [ + &["status"], + &["snapshot_id"], + &["data_file"], + &["data_file", "file_path"], + &["data_file", "file_format"], + &["data_file", "partition"], + &["data_file", "record_count"], + &["data_file", "file_size_in_bytes"], + ]; + assert_reads_without_each_field( + &metadata, + &full_schema, + &full_value, + &full_entry, + &optional, + &required, + ); + } + + #[test] + fn test_parse_manifest_matches_partition_fields_by_name() { + // The writer orders the partition fields differently from the spec, omits + // `name`, and adds a field the spec doesn't have. + let metadata = writer_schema_test_metadata(); + let writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + {"name": "extra", "type": ["null", "int"], "default": null, "field-id": 1003}, + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + ])); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry( + serde_json::json!({"score": 2.5, "extra": 9, "id": 5}), + )]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let partition = + Struct::from_iter([Some(Literal::long(5)), None, Some(Literal::double(2.5))]); + assert_eq!( + manifest, + Manifest::new(metadata, vec![expected_entry(partition)]) + ); + } + + #[test] + fn test_parse_manifest_promotes_writer_types() { + // Avro promotes int to long and float to double. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "int"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "float"], "default": null, "field-id": 1002}, + ])); + for field in ["record_count", "file_size_in_bytes"] { + writer_schema_field(&mut writer_schema, &["data_file", field])["type"] = + serde_json::json!("int"); + } + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry( + serde_json::json!({"id": 5, "name": "a", "score": 2.5}), + )]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let partition = Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ]); + assert_eq!( + manifest, + Manifest::new(metadata, vec![expected_entry(partition)]) + ); + } + + #[test] + fn test_parse_manifest_reads_required_writer_fields_as_optional() { + // Fields that are optional in the reader schema are written without a + // union. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": "long", "field-id": 1000}, + {"name": "name", "type": "string", "field-id": 1001}, + {"name": "score", "type": "double", "field-id": 1002}, + ])); + *writer_schema_field(&mut writer_schema, &["snapshot_id"]) = + serde_json::json!({"name": "snapshot_id", "type": "long", "field-id": 1}); + writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"name": "sort_order_id", "type": "int", "field-id": 140})); + let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5})); + entry["data_file"]["sort_order_id"] = serde_json::json!(3); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let mut expected = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + expected.data_file.sort_order_id = Some(3); + assert_eq!(manifest, Manifest::new(metadata, vec![expected])); + } + + #[test] + fn test_parse_manifest_reads_union_writer_fields_as_required() { + // A field that is required in the reader schema is written as a union with + // null. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + ])); + writer_schema_field(&mut writer_schema, &["data_file", "record_count"])["type"] = + serde_json::json!(["null", "long"]); + let partition = serde_json::json!({"id": 5, "name": "a", "score": 2.5}); + let bs = + write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry(partition.clone())]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let expected = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + assert_eq!(manifest, Manifest::new(metadata.clone(), vec![expected])); + + let mut entry = v2_entry(partition); + entry["data_file"]["record_count"] = Value::Null; + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let err = Manifest::parse_avro(&bs).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_parse_manifest_with_fixed_uuid_partition() { + // The spec stores a uuid in Avro as a 16-byte fixed with logical type uuid. + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "u", Type::Primitive(PrimitiveType::Uuid)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("u", "u", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let metadata = ManifestMetadata { + schema_id: 0, + schema, + partition_spec, + content: ManifestContentType::Data, + format_version: FormatVersion::V2, + }; + let writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "u", "default": null, "field-id": 1000, "type": ["null", { + "type": "fixed", "name": "uuid_fixed", "size": 16, "logicalType": "uuid", + }]}, + ])); + let uuid = uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7); + // `Value::resolve` doesn't convert a JSON string to a fixed uuid, so the + // partition value is set as an Avro value. + let mut entry = AvroValue::try_from(v2_entry(serde_json::json!({}))).unwrap(); + let AvroValue::Map(fields) = &mut entry else { + unreachable!("a JSON object converts to an Avro map"); + }; + let Some(AvroValue::Map(data_file)) = fields.get_mut("data_file") else { + unreachable!("a JSON object converts to an Avro map"); + }; + data_file.insert( + "partition".to_string(), + AvroValue::Map(HashMap::from([("u".to_string(), AvroValue::Uuid(uuid))])), + ); + let bs = write_avro_values_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let expected = expected_entry(Struct::from_iter([Some(Literal::uuid(uuid))])); + assert_eq!(manifest, Manifest::new(metadata, vec![expected])); + } + + #[test] + fn test_parse_manifest_ignores_record_names_and_unknown_fields() { + // Record names differ from the ones Java writes, and the writer has fields + // the reader schema doesn't. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + ])); + writer_schema["name"] = serde_json::json!("entry"); + writer_schema["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"name": "unknown", "type": "string"})); + let data_file = &mut writer_schema_field(&mut writer_schema, &["data_file"])["type"]; + data_file["name"] = serde_json::json!("file"); + data_file["fields"][3]["type"]["name"] = serde_json::json!("part"); + let data_file_fields = data_file["fields"].as_array_mut().unwrap(); + data_file_fields.push(serde_json::json!({ + "name": "column_sizes", + "field-id": 108, + "default": null, + "type": ["null", { + "type": "array", + "logicalType": "map", + "items": { + "type": "record", + "name": "sizes", + "fields": [ + {"name": "key", "type": "int", "field-id": 117}, + {"name": "value", "type": "long", "field-id": 118}, + ], + }, + }], + })); + data_file_fields.push( + serde_json::json!({"name": "block_size_in_bytes", "type": "long", "field-id": 105}), + ); + let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5})); + entry["unknown"] = serde_json::json!("ignored"); + entry["data_file"]["column_sizes"] = serde_json::json!([{"key": 1, "value": 40}]); + entry["data_file"]["block_size_in_bytes"] = serde_json::json!(64); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let mut expected = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + expected.data_file.column_sizes = HashMap::from([(1, 40)]); + assert_eq!(manifest, Manifest::new(metadata, vec![expected])); + } + + #[test] + fn test_parse_manifest_equality_ids_written_as_long() { + // PyIceberg wrote `equality_ids` as `array` before + // apache/iceberg-python#3842. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + ])); + writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": "equality_ids", + "field-id": 135, + "default": null, + "type": ["null", {"type": "array", "items": "long", "element-id": 136}], + })); + let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5})); + entry["data_file"]["equality_ids"] = serde_json::json!([1, 2]); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let mut expected = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + expected.data_file.equality_ids = Some(vec![1, 2]); + assert_eq!(manifest, Manifest::new(metadata, vec![expected])); + } + + #[test] + fn test_parse_manifest_rejects_equality_id_above_int_range() { + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + ])); + writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": "equality_ids", + "field-id": 135, + "default": null, + "type": ["null", {"type": "array", "items": "long", "element-id": 136}], + })); + let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5})); + entry["data_file"]["equality_ids"] = serde_json::json!([i64::from(i32::MAX) + 1]); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let err = Manifest::parse_avro(&bs).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_parse_manifest_written_by_pyiceberg() { + let bs = fs::read(format!( + "{}/testdata/manifests/pyiceberg-v2-data.avro", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "category", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::optional(3, "score", Type::Primitive(PrimitiveType::Double)) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("category", "category", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let metadata = ManifestMetadata { + schema_id: 0, + schema, + partition_spec, + content: ManifestContentType::Data, + format_version: FormatVersion::V2, + }; + let entries = (0..2) + .map(|i: i64| { + let category = format!("c{i}"); + ManifestEntry { + status: ManifestStatus::Added, + snapshot_id: Some(7), + sequence_number: Some(1), + file_sequence_number: Some(1), + data_file: DataFile { + content: DataContentType::Data, + file_path: format!( + "s3://bucket/table/data/category={category}/0000{i}.parquet" + ), + file_format: DataFileFormat::Parquet, + partition: Struct::from_iter([Some(Literal::string(&category))]), + record_count: 100 + i as u64, + file_size_in_bytes: 1000 + i as u64, + column_sizes: HashMap::from([ + (1, 10 + i as u64), + (2, 20 + i as u64), + (3, 30 + i as u64), + ]), + value_counts: HashMap::from([ + (1, 100 + i as u64), + (2, 100 + i as u64), + (3, 100 + i as u64), + ]), + null_value_counts: HashMap::from([(1, 0), (2, i as u64), (3, 1)]), + nan_value_counts: HashMap::from([(3, i as u64)]), + lower_bounds: HashMap::from([ + (1, Datum::long(i)), + (2, Datum::string(&category)), + ]), + upper_bounds: HashMap::from([ + (1, Datum::long(i + 50)), + (2, Datum::string(&category)), + ]), + key_metadata: None, + split_offsets: Some(vec![4]), + equality_ids: None, + sort_order_id: Some(0), + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + }, + } + }) + .collect(); + assert_eq!(manifest, Manifest::new(metadata, entries)); + } + + #[test] + fn test_parse_manifest_with_repeated_named_type_definitions() { + let bs = fs::read(format!( + "{}/testdata/manifests/repeated-decimal-type-definitions.avro", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + assert_eq!( + *manifest.entries()[0].data_file().partition(), + Struct::from_iter([Some(Literal::decimal(12345)), Some(Literal::decimal(-678))]) + ); + } + + #[test] + fn test_parse_manifest_with_repeated_named_type_definitions_reports_original_error() { + let bs = fs::read(format!( + "{}/testdata/manifests/repeated-decimal-type-definitions.avro", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + // Cut the header's sync marker in half, so the rewritten header parses + // its schema but fails to read the marker. The file ends with the marker. + let marker = &bs[bs.len() - 16..]; + let header_marker = bs.windows(16).position(|w| w == marker).unwrap(); + let truncated = &bs[..header_marker + 8]; + let log_dir = TempDir::new().unwrap(); + let log_path = log_dir.path().join("log"); + let subscriber = tracing_subscriber::fmt() + .with_max_level(Level::DEBUG) + .with_writer(Arc::new(fs::File::create(&log_path).unwrap())) + .finish(); + + let err = with_default(subscriber, || Manifest::parse_avro(truncated)).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + let message = err.to_string(); + assert!( + message.contains("Two named schema defined for same fullname"), + "{message}" + ); + assert!(message.contains("Failed to read marker bytes"), "{message}"); + // The fallback logs only manifests that it reads. + let log = fs::read_to_string(&log_path).unwrap(); + assert!(!log.contains("more than once"), "{log}"); + } } diff --git a/crates/iceberg/src/spec/manifest/reader.rs b/crates/iceberg/src/spec/manifest/reader.rs index e7044a56bc..a056d207c4 100644 --- a/crates/iceberg/src/spec/manifest/reader.rs +++ b/crates/iceberg/src/spec/manifest/reader.rs @@ -51,7 +51,8 @@ impl ManifestReader { None => input_file.read().await?, }; - let (metadata, mut entries) = Manifest::try_from_avro_bytes(&bytes)?; + let (metadata, mut entries) = + Manifest::try_from_avro_bytes(&bytes, Some(&manifest_file.manifest_path))?; for entry in &mut entries { entry.inherit_data(manifest_file); diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index 15608a63f8..a79312dfab 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -432,7 +432,7 @@ impl ManifestWriter { // Manifest schema did not change between V2 and V3 FormatVersion::V2 | FormatVersion::V3 => manifest_schema_v2(&partition_type)?, }; - let mut avro_writer = AvroWriter::new(&avro_schema, Vec::new()); + let mut avro_writer = AvroWriter::new(&avro_schema, Vec::new())?; avro_writer.add_user_metadata( "schema".to_string(), to_vec(table_schema) @@ -479,7 +479,7 @@ impl ManifestWriter { } }; - avro_writer.append(value)?; + avro_writer.append_value(value)?; } let content = avro_writer.into_inner()?; @@ -580,8 +580,12 @@ mod tests { use tempfile::TempDir; use super::*; + use crate::avro::define_named_types_repeatedly; use crate::io::FileIO; - use crate::spec::{DataFileFormat, Manifest, NestedField, PrimitiveType, Schema, Struct, Type}; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Literal, Manifest, NestedField, + PrimitiveType, Schema, Struct, Transform, Type, + }; #[tokio::test] async fn test_add_delete_existing() { @@ -819,4 +823,102 @@ mod tests { ManifestContentType::Deletes, ); } + + /// Writes a manifest partitioned by identity on two columns of `field_type`, + /// reads it back, and returns the manifest file. + async fn roundtrip_two_partition_fields_of_type( + field_type: Type, + values: [Literal; 2], + ) -> Vec { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "a", field_type.clone()).into(), + NestedField::optional(2, "b", field_type).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("a", "a", Transform::Identity) + .unwrap() + .add_partition_field("b", "b", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition = Struct::from_iter(values.map(Some)); + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("manifest.avro"); + let output_file = FileIO::new_with_fs() + .new_output(path.to_str().unwrap()) + .unwrap(); + let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec) + .build_v2_data(); + writer + .add_file( + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("s3://bucket/table/data/a.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .partition(partition.clone()) + .record_count(1) + .file_size_in_bytes(1) + .partition_spec_id(0) + .build() + .unwrap(), + 1, + ) + .unwrap(); + writer.write_manifest_file().await.unwrap(); + let bs = fs::read(&path).unwrap(); + + // Also read the manifest as iceberg-rust wrote it before defining each + // named type once. + for bs in [bs.clone(), define_named_types_repeatedly(&bs)] { + let manifest = Manifest::parse_avro(&bs).unwrap(); + + assert_eq!(*manifest.entries()[0].data_file().partition(), partition); + } + bs + } + + #[tokio::test] + async fn test_write_manifest_header_marks_int_keyed_maps() { + // Java and PyIceberg read an array of key-value records as a map only if + // it has `logicalType: map`. apache-avro 0.22 drops the attribute when it + // parses a schema (https://github.com/apache/avro-rs/issues/654), so this + // checks the schema JSON in the written header. + let bs = roundtrip_two_partition_fields_of_type(Type::Primitive(PrimitiveType::Int), [ + Literal::int(1), + Literal::int(2), + ]) + .await; + + let file = String::from_utf8_lossy(&bs); + // `column_sizes`, `value_counts`, `null_value_counts`, + // `nan_value_counts`, `lower_bounds`, and `upper_bounds`. + assert_eq!(file.matches(r#""logicalType":"map""#).count(), 6); + } + + #[tokio::test] + async fn test_write_manifest_with_repeated_decimal_partition_type() { + roundtrip_two_partition_fields_of_type( + Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }), + [Literal::decimal(12345), Literal::decimal(-678)], + ) + .await; + } + + #[tokio::test] + async fn test_write_manifest_with_repeated_fixed_partition_type() { + roundtrip_two_partition_fields_of_type(Type::Primitive(PrimitiveType::Fixed(4)), [ + Literal::fixed(vec![1, 2, 3, 4]), + Literal::fixed(vec![5, 6, 7, 8]), + ]) + .await; + } } diff --git a/crates/iceberg/src/spec/manifest_list/mod.rs b/crates/iceberg/src/spec/manifest_list/mod.rs index 25dd5bb93e..24a4debf9e 100644 --- a/crates/iceberg/src/spec/manifest_list/mod.rs +++ b/crates/iceberg/src/spec/manifest_list/mod.rs @@ -61,7 +61,9 @@ impl ManifestList { pub fn parse_with_version(bs: &[u8], version: FormatVersion) -> Result { match version { FormatVersion::V1 => { - let reader = Reader::with_schema(&MANIFEST_LIST_AVRO_SCHEMA_V1, bs)?; + let reader = Reader::builder(bs) + .reader_schema(&MANIFEST_LIST_AVRO_SCHEMA_V1) + .build()?; let values = Value::Array(reader.collect::, _>>()?); from_value::<_serde::ManifestListV1>(&values)?.try_into() } @@ -265,7 +267,7 @@ mod test { let manifest_entry: ManifestFileV2 = manifest_list.entries[0].clone().try_into().unwrap(); let mut writer = - Writer::with_codec(&MANIFEST_LIST_AVRO_SCHEMA_V2, Vec::new(), Codec::Snappy); + Writer::with_codec(&MANIFEST_LIST_AVRO_SCHEMA_V2, Vec::new(), Codec::Snappy).unwrap(); writer.append_ser(manifest_entry).unwrap(); let bs = writer.into_inner().unwrap(); diff --git a/crates/iceberg/src/spec/manifest_list/writer.rs b/crates/iceberg/src/spec/manifest_list/writer.rs index 5ce2cfb259..108daa951e 100644 --- a/crates/iceberg/src/spec/manifest_list/writer.rs +++ b/crates/iceberg/src/spec/manifest_list/writer.rs @@ -149,7 +149,9 @@ impl ManifestListWriter { FormatVersion::V2 => &MANIFEST_LIST_AVRO_SCHEMA_V2, FormatVersion::V3 => &MANIFEST_LIST_AVRO_SCHEMA_V3, }; - let mut avro_writer = Writer::new(avro_schema, Vec::new()); + let mut avro_writer = Writer::new(avro_schema, Vec::new()).expect( + "Manifest list Avro schemas should resolve because they refer to no named types.", + ); for (key, value) in metadata { avro_writer .add_user_metadata(key, value) diff --git a/crates/iceberg/src/spec/values/serde.rs b/crates/iceberg/src/spec/values/serde.rs index 896235b1e3..a9c6115171 100644 --- a/crates/iceberg/src/spec/values/serde.rs +++ b/crates/iceberg/src/spec/values/serde.rs @@ -18,6 +18,8 @@ //\! Serialization and deserialization support for Iceberg values pub(crate) mod _serde { + use std::collections::HashMap; + use serde::de::Visitor; use serde::ser::{SerializeMap, SerializeSeq, SerializeStruct}; use serde::{Deserialize, Serialize}; @@ -27,7 +29,7 @@ pub(crate) mod _serde { use crate::Error; use crate::error::invalid_data; use crate::spec::values::{Literal, Map, PrimitiveLiteral, Struct}; - use crate::spec::{MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, PrimitiveType, Type}; + use crate::spec::{MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, PrimitiveType, StructType, Type}; #[derive(SerializeDerive, DeserializeDerive, Debug)] #[serde(transparent)] @@ -44,6 +46,53 @@ pub(crate) mod _serde { pub fn try_into(self, ty: &Type) -> Result, Error> { self.0.try_into(ty) } + + /// Matches the fields of a record to `struct_type` by name, the way Avro + /// schema resolution matches record fields. The result has + /// `struct_type`'s fields in its order, with null for an optional field the + /// record lacks. Record fields that `struct_type` lacks are dropped. Values + /// other than records are returned unchanged. + /// + /// Fields aren't matched by field ID. A writer that stores a field under + /// another name, such as Java replacing characters that Avro names don't + /// allow, reads as a missing field. + pub fn project_by_name(self, struct_type: &StructType) -> Result { + let (mut required, optional) = match self.0 { + RawLiteralEnum::Record(Record { required, optional }) => (required, optional), + other => return Ok(Self(other)), + }; + required.extend( + optional + .into_iter() + .map(|(name, value)| (name, value.unwrap_or(RawLiteralEnum::Null))), + ); + let fields = struct_type.fields(); + let required = if required.len() == fields.len() + && required + .iter() + .zip(fields) + .all(|((name, _), field)| *name == field.name) + { + required + } else { + let mut values: HashMap = required.into_iter().collect(); + fields + .iter() + .map(|field| match values.remove(&field.name) { + Some(value) => Ok((field.name.clone(), value)), + None if field.required => Err(invalid_data!( + "Record has no value for required field {}", + field.name + )), + None => Ok((field.name.clone(), RawLiteralEnum::Null)), + }) + .collect::>()? + }; + Ok(Self(RawLiteralEnum::Record(Record { + required, + optional: Vec::new(), + }))) + } } #[derive(SerializeDerive, Clone, Debug)] diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index ff51f318e3..c756bc3f8e 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -64,10 +64,13 @@ fn check_avro_bytes_serde(input: Vec, expected_datum: Datum, expected_type: let datum = Datum::try_from_bytes(&bytes, expected_type.clone()).unwrap(); assert_eq!(datum, expected_datum); - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); + let mut writer = apache_avro::Writer::new(&schema, Vec::new()).unwrap(); writer.append_ser(datum.to_bytes().unwrap()).unwrap(); let encoded = writer.into_inner().unwrap(); - let reader = apache_avro::Reader::with_schema(&schema, &*encoded).unwrap(); + let reader = apache_avro::Reader::builder(&*encoded) + .reader_schema(&schema) + .build() + .unwrap(); for record in reader { let result = apache_avro::from_value::(&record.unwrap()).unwrap(); @@ -86,7 +89,7 @@ fn check_convert_with_avro(expected_literal: Literal, expected_type: &Type) { let struct_type = Type::Struct(StructType::new(fields)); let struct_literal = Literal::Struct(Struct::from_iter(vec![Some(expected_literal.clone())])); - let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()); + let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()).unwrap(); let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); writer.append_ser(raw_literal).unwrap(); let encoded = writer.into_inner().unwrap(); @@ -110,7 +113,7 @@ fn check_serialize_avro(literal: Literal, ty: &Type, expect_value: Value) { let avro_schema = schema_to_avro_schema("test", &schema).unwrap(); let struct_type = Type::Struct(StructType::new(fields)); let struct_literal = Literal::Struct(Struct::from_iter(vec![Some(literal.clone())])); - let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()); + let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()).unwrap(); let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); let value = to_value(raw_literal) .unwrap() @@ -924,7 +927,7 @@ fn check_convert_with_avro_map(expected_literal: Literal, expected_type: &Type) let struct_type = Type::Struct(StructType::new(fields)); let struct_literal = Literal::Struct(Struct::from_iter(vec![Some(expected_literal.clone())])); - let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()); + let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()).unwrap(); let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); writer.append_ser(raw_literal).unwrap(); let encoded = writer.into_inner().unwrap(); @@ -1719,3 +1722,60 @@ fn test_datum_to_decimal_rejects_scale_change() { .contains("Decimal scale conversion is not supported") ); } + +#[test] +fn test_raw_literal_project_by_name_reorders_and_fills_fields() { + let written = StructType::new(vec![ + NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "b", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "extra", Primitive(PrimitiveType::Int)).into(), + ]); + let raw = RawLiteral::try_from( + Literal::Struct(Struct::from_iter([ + Some(Literal::int(1)), + Some(Literal::int(2)), + Some(Literal::int(3)), + ])), + &Type::Struct(written), + ) + .unwrap(); + let expected = StructType::new(vec![ + NestedField::optional(4, "c", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "b", Primitive(PrimitiveType::Int)).into(), + NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(), + ]); + + let projected = raw + .project_by_name(&expected) + .unwrap() + .try_into(&Type::Struct(expected)) + .unwrap(); + + assert_eq!( + projected, + Some(Literal::Struct(Struct::from_iter([ + None, + Some(Literal::int(2)), + Some(Literal::int(1)), + ]))) + ); +} + +#[test] +fn test_raw_literal_project_by_name_rejects_missing_required_field() { + let written = StructType::new(vec![ + NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(), + ]); + let raw = RawLiteral::try_from( + Literal::Struct(Struct::from_iter([Some(Literal::int(1))])), + &Type::Struct(written), + ) + .unwrap(); + let expected = StructType::new(vec![ + NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "b", Primitive(PrimitiveType::Int)).into(), + ]); + + let err = raw.project_by_name(&expected).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); +} diff --git a/crates/iceberg/testdata/manifests/README.md b/crates/iceberg/testdata/manifests/README.md new file mode 100644 index 0000000000..6b68b71ed0 --- /dev/null +++ b/crates/iceberg/testdata/manifests/README.md @@ -0,0 +1,99 @@ + + + +# Manifest test data + +`repeated-decimal-type-definitions.avro` is a V2 data manifest written by +iceberg-rust at commit `ecba0b5`, which used `apache-avro` 0.21. Its table schema +has two optional `decimal(10, 2)` columns, `d1` (id 1) and `d2` (id 2), and its +partition spec has an identity partition on each. It holds one entry for +`s3://b/t/a.parquet` with partition values `123.45` and `-6.78`. + +The Avro schema in its header defines the named fixed type `decimal_10_2` twice. +The Avro specification allows only one definition of each name, and +`apache-avro` 0.22 rejects the header, but iceberg-rust wrote manifests like +this one before it moved to 0.22. Tests use it to check that such manifests +still read. + +To regenerate it, check out a commit that uses `apache-avro` 0.21, write the +manifest described above with `ManifestWriterBuilder::build_v2_data`, and copy +the file here. + +`pyiceberg-v2-data.avro` is a V2 data manifest written by PyIceberg 0.12.0 +(apache/iceberg-python commit `0d584073`) with deflate compression. Its table +schema has optional columns `id` (long), `category` (string), and `score` +(double), and its partition spec has an identity partition on `category`. It +holds two entries with column metrics. Its Avro schema is PyIceberg's, so tests +use it to check reading a manifest whose writer schema iceberg-rust didn't +generate. + +To regenerate it, run this script with PyIceberg installed, passing the output +path: + +```python +import sys + +from pyiceberg.io.pyarrow import PyArrowFileIO +from pyiceberg.manifest import DataFile, DataFileContent, FileFormat, ManifestEntry, ManifestEntryStatus, write_manifest +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema +from pyiceberg.transforms import IdentityTransform +from pyiceberg.typedef import Record +from pyiceberg.types import DoubleType, LongType, NestedField, StringType + +SCHEMA = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "category", StringType(), required=False), + NestedField(3, "score", DoubleType(), required=False), +) +SPEC = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="category"), spec_id=0) + + +def entry(i: int) -> ManifestEntry: + data_file = DataFile.from_args( + content=DataFileContent.DATA, + file_path=f"s3://bucket/table/data/category=c{i}/{i:05d}.parquet", + file_format=FileFormat.PARQUET, + partition=Record(f"c{i}"), + record_count=100 + i, + file_size_in_bytes=1000 + i, + column_sizes={1: 10 + i, 2: 20 + i, 3: 30 + i}, + value_counts={1: 100 + i, 2: 100 + i, 3: 100 + i}, + null_value_counts={1: 0, 2: i, 3: 1}, + nan_value_counts={3: i}, + lower_bounds={1: (i).to_bytes(8, "little"), 2: f"c{i}".encode()}, + upper_bounds={1: (i + 50).to_bytes(8, "little"), 2: f"c{i}".encode()}, + split_offsets=[4], + sort_order_id=0, + ) + return ManifestEntry.from_args( + status=ManifestEntryStatus.ADDED, snapshot_id=7, sequence_number=1, file_sequence_number=1, data_file=data_file + ) + + +def main(path: str) -> None: + with write_manifest(2, SPEC, SCHEMA, PyArrowFileIO().new_output(path), snapshot_id=7, avro_compression="deflate") as writer: + for i in range(2): + writer.add_entry(entry(i)) + + +if __name__ == "__main__": + main(sys.argv[1]) +``` diff --git a/crates/iceberg/testdata/manifests/pyiceberg-v2-data.avro b/crates/iceberg/testdata/manifests/pyiceberg-v2-data.avro new file mode 100644 index 0000000000000000000000000000000000000000..c8c6866449aade34c2f7e53e0f76e521d658f991 GIT binary patch literal 4787 zcmb_gU1%It6y9W05c`8jksw8`(>|!3)ScOyY=Yo}O*GP0u@Qu_jCW@4He+XJrg!e7 zjb)+LQmj^?wGVj^6zPLc;)_1F(qdi$B7G?8i&_xGzKNh1^qhNlW_NbCYqBncke!`- z&Ue0Z&Ueqri}vdyJGRBnBCFHyil;TP95HQ1iv>>{q3K%9W4;?}Ge=Kro~z9mx)#tn zz5$=~^ld*3j=?9+P9z>@@LG-fF>C1RjG3Krs9?uJzT7$qe!PBX%h}o32{~&Wbi>5< z@pGrss;;SMCJ=8)2cRqv9>l>XWH9g?_A;$*#g5fSD$~&}ImJl{X2MzygO^K~lFF z?=V{))NIZ3nHBf7y?;>K$hd_LI>4GOgBwjjGl?CBjB*<`nFZu9bbvo?4*U{T9|RLQ zLooL`3Auy62$T~L+L1TOMnuJ8rz0rUR=U#;%X}RSp-`a!$ty3hjQ58K@PqhcgcYMU!<*m#;xGkO@t`p0SHFYFZ`TjG#wRlD>@E$@eA zXOXZ*l!cgh${D6D??NUr_TFHY7g+YPV4d<{l-f&Esd*OWLMW(D&;~&p=tkMdb-WY$ zNj3sT}ep>L;@4(<5p8?U^ttjVh+z507~c^%vwVNwhkRv_=)4=d}CIDVmWEBG z>>V8n)A17*q)6UJga$4-Ob_R|#hc3NN3hXLAX){v$yzlHB-k9BVAB|YU~_l`BVR=F z1vQKcm4g$|pf8!_sbN-YxdLRCrw1pqT*;x|P~(@Uhe;+5L#PK(mx|R)wUt8T>Gi{t zj9WJF4ihUd*)B-tQ^TazoWwev%A9Ok2+Ya;)b_k}%&FmVE4d<)t2~KG6y2M|p;@(|@za@C7OxKyn=6BGxXtVo(WLb+`g%&sI&VXZ=u-s@_d4II84 z$w?HD_Ek7?1rIw3g*TRAhw4VS!B*`+IN6R+E^YnToWngG28@^_IC#RC;WEK^-3u7u z;YpGL#PKcr+Xa^b77v*1;(Z;Qha4#9F7!_KTDZU0B`H^$eh}vmz^EE=7USX!w{#3H zV7UPoxcb1L?8OpF9tdYO#&MT)t$VF##{RZBIjezmFX;zq4z9OlB6A1g>JmTQjUMY> zjvhzhrfSxg)38{if>5|06^$`O*MzEO!gOTiurz8j_{cvIZ%8knh*6-Z7 zeCzkWKY8JMx7a8?@Xg-m^(XX06F+W^w%xxi*-q{G$;$OpXFs~|-u$`Qx7N;&tX8hC z@;5Kt9ozl(+O@9=dsbF%kB_gsx-vHY@tG~3Zr5KsrxzB!9@#TKUYK2dxC_r88^`l( zqex1(dH{O*=BWz^=;G|VYwrV~o2&etOLuo7piyn*+St|)3cHJApBKJ3yXE4gEoXkM OoEa+?3kNTBq4^i@F9Rk3 literal 0 HcmV?d00001 diff --git a/crates/iceberg/testdata/manifests/repeated-decimal-type-definitions.avro b/crates/iceberg/testdata/manifests/repeated-decimal-type-definitions.avro new file mode 100644 index 0000000000000000000000000000000000000000..598268c03003ddd9ba71cdf7ae46a69146efe0d8 GIT binary patch literal 3865 zcmd5yC1ri!jfaD~*vw7Ja1vL*r0Vo=ccW23veK9i|FvvX>1wTLm zUx36XP{A*uaA)@6S+8S<#RX0iujjtbJ?GBd2cuW(SFX`hk#75G!2C-|9z`2|nkbg2 zO955%Mn;9=il@nz%$WbN_u@U0X)b)W#RD{S`zYir3W_g|-2BoyC!9A4Y^j*(F->^Lq#^*77&o&ok4QyTF5#!LP@dFP77%E7tFsBsIciixRAo^|d`JkgeeXZdp|K?BJ%JjH=_QG4PKAz6~44u@W{mS{hXMZ_3)Xm1|km zYOHO}FVl7A7g{LO$fP1e?Ih%HK@2q?OBoQ?A=;PZ??797nmaGKPopP=*2+`*06c^f zAd?KJ%_WQtm=VroAUR$fj7{dmG($_saX1^KfO<8x51@AJbg6wXo7NPlb^|de z04Vg=wcA1TbS4Jn6#{@=4Z!>{vAFrzo=Qubk8vjeU~)_^vxFuquira_`n^F10(dQ6 zE+T+8=v2RFD=bRiFhNEzXM&ZGAMfklA9Sc4?^|t|>3ucKtg8|qbSfV2SyA#VTW#@L z`Cb?0V_THVi}$)oAKUZ-C6bsas&|~4W!1<>1>y^DSkD%#%p$Hx8itaoixa7u=qzF} zET_f3?|F-dw`X6Ju&j`nI%uCZl<+jO}=md$m;ybGFrqrI%u zX{aIVPExqB+&S*#N&~0!F?<;EK;l0O?p5A`~zpgoh-r3pN@8|mL^bdY8 IpINtm03fg3 Date: Fri, 9 Oct 2026 16:15:35 +0000 Subject: [PATCH 46/63] feat(reader): page-index pruning for FIXED_LEN_BYTE_ARRAY decimals (#3328) Decimals with precision > 18 are stored as FIXED_LEN_BYTE_ARRAY, which the page-index evaluator skipped, so inequality and IN predicates on them got no page pruning (over-selection only, never dropped rows). Decode those bounds with i128_from_be_bytes, matching the row-group statistics path. Guard against truncated ColumnIndex bounds: a bound whose width differs from the column's declared type_length signals a column_index_truncate_length truncation, and a truncated byte prefix no longer preserves the two's complement numeric ordering of decimals. Reject it so the column skips pruning rather than decode the wrong number. parquet-rs never truncates decimal column indexes, so this only defends against files written by other implementations. Benchmark A throwaway benchmark on a 1,000,000-row single-column decimal(30,2) file (precision > 18 -> FIXED_LEN_BYTE_ARRAY), 100 pages in one row group, read fully in memory so timing is decode-only. A `col > threshold` predicate sweeps selectivity. "read full file" is today's behavior (FLBA decimals skip page pruning); "read ms" applies the page-index RowSelection this change produces. 1,000,000 rows in 100 pages, 1 row group read full file : 3.431 ms predicate (keep) | rows read | eval us | read ms | speedup 1% | 10,000 | 17.5 | 0.038 | 90.0x 10% | 100,000 | 17.1 | 0.330 | 10.4x 50% | 500,000 | 16.6 | 1.629 | 2.1x 100% | 1,000,000 | 16.3 | 3.397 | 1.0x The win tracks predicate selectivity; eval() adds a constant ~17 us per row group per scan, negligible against the millisecond read. The benefit only materializes where page-index row selection runs: MoR tables or with_row_selection_enabled(true). Reproducer in the PR description. Follow-up to #3314. --- .../src/expr/visitors/page_index_evaluator.rs | 322 +++++++++++++++++- 1 file changed, 314 insertions(+), 8 deletions(-) diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index d494e5426f..88d1eea9ff 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -28,6 +28,7 @@ use parquet::file::page_index::offset_index::OffsetIndexMetaData; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; +use crate::spec::decimal_utils::i128_from_be_bytes; use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType, Schema}; use crate::{Error, ErrorKind, Result}; @@ -167,6 +168,17 @@ impl<'a> PageIndexEvaluator<'a> { return self.select_all_rows(); }; + // Declared byte width of a FIXED_LEN_BYTE_ARRAY column, used to detect + // truncated page-index bounds when decoding decimals. `None` when the + // column metadata is absent or reports no fixed width. + let type_length = self + .row_group_metadata + .columns() + .get(parquet_column_index) + .map(|column| column.column_descr().type_length()) + .filter(|&len| len > 0) + .map(|len| len as usize); + let row_counts = { // Caches row count calculations for columns that appear multiple times in // the predicate @@ -194,6 +206,7 @@ impl<'a> PageIndexEvaluator<'a> { field_id, predicate, field_type, + type_length, column_index, &row_counts, )? @@ -242,6 +255,7 @@ impl<'a> PageIndexEvaluator<'a> { field_id: i32, predicate: F, field_type: &PrimitiveType, + type_length: Option, column_index: &ColumnIndexMetaData, row_counts: &[usize], ) -> Result>> @@ -394,14 +408,89 @@ impl<'a> PageIndexEvaluator<'a> { Ok(page_filter) } + ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(idx) => { + // Parquet stores Iceberg decimals with precision > 18 as + // FIXED_LEN_BYTE_ARRAY. Other field types on such a column + // (fixed, uuid) aren't decoded here, so skip page pruning for + // them rather than risk pruning pages that match. + if !matches!(field_type, PrimitiveType::Decimal { .. }) { + tracing::debug!( + field_id, + %field_type, + "Skipping page-index pruning: FIXED_LEN_BYTE_ARRAY column index on a non-decimal field" + ); + return Ok(None); + } + + let Some(type_length) = type_length else { + tracing::debug!( + field_id, + %field_type, + "Skipping page-index pruning: unknown FIXED_LEN_BYTE_ARRAY decimal column width" + ); + return Ok(None); + }; + + let mut page_filter = Vec::with_capacity(row_counts.len()); + for ((i, (min, max)), &row_count) in idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() + .zip(row_counts.iter()) + { + // A bound that won't decode (e.g. a min/max stat truncated + // by column_index_truncate_length, whose byte prefix no + // longer preserves the two's-complement decimal ordering) + // means this column's page index can't be trusted, so skip + // pruning for the whole column rather than abort the scan. + let (min, max) = match ( + min.map(|val| { + Self::fixed_len_byte_array_decimal_bound_to_datum( + field_type, + val, + type_length, + ) + }) + .transpose(), + max.map(|val| { + Self::fixed_len_byte_array_decimal_bound_to_datum( + field_type, + val, + type_length, + ) + }) + .transpose(), + ) { + (Ok(min), Ok(max)) => (min, max), + (Err(err), _) | (_, Err(err)) => { + tracing::debug!( + field_id, + %field_type, + page_index = i, + %err, + "Skipping page-index pruning: undecodable FIXED_LEN_BYTE_ARRAY decimal page bound" + ); + return Ok(None); + } + }; + + page_filter.push(predicate( + min, + max, + PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), + )?); + } + + Ok(page_filter) + } // Column index types we can't interpret: skip page pruning rather // than abort the scan. Row-group filtering and the Arrow row filter // still apply the predicate. - ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(_) | ColumnIndexMetaData::INT96(_) => { + ColumnIndexMetaData::INT96(_) => { tracing::debug!( field_id, %field_type, - "Skipping page-index pruning: unsupported FIXED_LEN_BYTE_ARRAY or INT96 column index" + "Skipping page-index pruning: unsupported INT96 column index" ); return Ok(None); } @@ -455,6 +544,40 @@ impl<'a> PageIndexEvaluator<'a> { } } + /// Converts a `FIXED_LEN_BYTE_ARRAY` decimal page bound into a [`Datum`]. + /// Parquet stores Iceberg decimals with precision > 18 as big-endian + /// two's-complement `FIXED_LEN_BYTE_ARRAY` of width `type_length`. + /// + /// Returns an error for a bound whose width differs from `type_length`: that + /// signals a `column_index_truncate_length` truncation, and a truncated byte + /// prefix decodes to the wrong number under two's-complement. + fn fixed_len_byte_array_decimal_bound_to_datum( + field_type: &PrimitiveType, + bytes: &[u8], + type_length: usize, + ) -> Result { + if bytes.len() != type_length { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Truncated FIXED_LEN_BYTE_ARRAY decimal page bound: {} bytes (expected {type_length})", + bytes.len() + ), + )); + } + + let value = i128_from_be_bytes(bytes).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Can't convert bytes to i128: {bytes:?}"), + ) + })?; + Ok(Datum::new( + field_type.clone(), + PrimitiveLiteral::Int128(value), + )) + } + fn visit_inequality( &mut self, reference: &BoundReference, @@ -1062,11 +1185,58 @@ mod tests { Ok((metadata, temp_file)) } + /// Creates a single-column `FixedSizeBinary(width)` parquet file whose column + /// index is truncated to `truncate_length` bytes (`< width`), producing + /// `FIXED_LEN_BYTE_ARRAY` page bounds narrower than the declared `type_length`. + /// Writes one 1024-row page per value in `pages`; each value must be `width` + /// bytes. + fn create_truncated_flba_file( + width: i32, + truncate_length: usize, + pages: &[&[u8]], + ) -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "col_fixed", + DataType::FixedSizeBinary(width), + true, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .set_column_index_truncate_length(Some(truncate_length)) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + for value in pages { + let array = Arc::new( + FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n(*value, 1024)).unwrap(), + ) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + /// Creates a single-column `Decimal128(precision, scale)` parquet file. - /// Parquet encodes precision <= 9 as `INT32` and precision 10..=18 as - /// `INT64`, so `precision` selects which page-index encoding is exercised. - /// Writes one 1024-row page per value in `unscaled`, so page bounds - /// partition the value range. + /// Parquet encodes precision <= 9 as `INT32`, precision 10..=18 as `INT64`, + /// and precision > 18 as `FIXED_LEN_BYTE_ARRAY`, so `precision` selects which + /// page-index encoding is exercised. Writes one 1024-row page per value in + /// `unscaled`, so page bounds partition the value range. fn create_decimal_parquet_file( precision: u8, scale: i8, @@ -1490,8 +1660,9 @@ mod tests { #[test] fn eval_skips_pruning_for_fixed_len_byte_array() -> Result<()> { // FIXED_LEN_BYTE_ARRAY page indexes back spec-conforming fixed, uuid, - // and decimal(P > 18) columns. The evaluator can't interpret those - // bounds, so it skips page pruning rather than abort the scan. + // and decimal(P > 18) columns. Only decimals are decoded; fixed and uuid + // bounds aren't, so the evaluator skips page pruning for them rather than + // abort the scan. let (metadata, _temp_file) = create_fixed_len_byte_array_parquet_file()?; let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); @@ -1880,6 +2051,141 @@ mod tests { Ok(()) } + #[test] + fn eval_inequality_prunes_fixed_len_byte_array_decimal_pages() -> Result<()> { + // precision 30 -> Parquet FIXED_LEN_BYTE_ARRAY page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(30, 2, &[100, 200, 300, 400])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. `> 2.50` keeps the pages whose + // upper bound exceeds 2.50 (3.00 and 4.00). + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(250, 2, 30)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(2048) + ]); + + Ok(()) + } + + #[test] + fn eval_in_prunes_fixed_len_byte_array_decimal_pages() -> Result<()> { + // precision 30 -> Parquet FIXED_LEN_BYTE_ARRAY page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(30, 2, &[100, 200, 300, 400])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. IN (2.00, 4.00) keeps only the + // pages whose single value is one of the literals. + let filter = Reference::new("col_decimal") + .is_in([decimal_datum(200, 2, 30)?, decimal_datum(400, 2, 30)?]) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(1024), + RowSelector::skip(1024), + RowSelector::select(1024), + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_negative_fixed_len_byte_array_decimal_pages() -> Result<()> { + // precision 30 -> Parquet FIXED_LEN_BYTE_ARRAY page bounds, spanning + // negative values. Negative decimals have a 0xff high byte in two's + // complement, so this only prunes correctly if the bounds are compared + // numerically rather than by raw byte order. + let (metadata, _temp_file) = create_decimal_parquet_file(30, 2, &[-400, -200, 100, 300])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + // Pages hold -4.00, -2.00, 1.00, 3.00. `>= -2.00` keeps the pages whose + // upper bound is at least -2.00, including page 1 whose bound equals it. + let filter = Reference::new("col_decimal") + .greater_than_or_equal_to(decimal_datum(-200, 2, 30)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(3072) + ]); + + Ok(()) + } + + #[test] + fn eval_skips_pruning_for_truncated_fixed_len_byte_array_decimal_bound() -> Result<()> { + // A non-Iceberg writer can store a decimal as a plain FIXED_LEN_BYTE_ARRAY + // and truncate its column index (parquet-rs truncates FLBA bounds but not + // decimal/float16 ones). A bound narrower than the column's declared + // type_length can't be decoded as an i128 without changing its value, so + // the evaluator skips page pruning for the column rather than aborting the + // file read. + // + // Page 0 is 0.00, page 1 is 2.55; with full-width bounds `> 1.00` would + // prune page 0. The 13-byte values are truncated to 4-byte column-index + // bounds (!= type_length 13), so neither page is pruned instead. + let page0 = [0u8; 13]; // unscaled 0 -> 0.00 + let mut page1 = [0u8; 13]; + page1[12] = 0xff; // unscaled 255 -> 2.55 + let (metadata, _temp_file) = create_truncated_flba_file(13, 4, &[&page0, &page1])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + + // The Iceberg schema calls the 13-byte column a Decimal(30,2). + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(100, 2, 30)?) // > 1.00 + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + // Truncated bounds -> no page pruning: every row survives. + assert_eq!(result, vec![RowSelector::select(2048)]); + + Ok(()) + } + /// Builds a decimal query [`Datum`] from an `unscaled` mantissa, matching /// the field's `scale` and `precision`. fn decimal_datum(unscaled: i128, scale: u32, precision: u32) -> Result { From 1f3bc34058e3f3e45336bc5428f37b92649a608b Mon Sep 17 00:00:00 2001 From: Feng Zhang Date: Fri, 9 Oct 2026 17:29:50 +0000 Subject: [PATCH 47/63] fix(spec): treat -0.0 and 0.0 as distinct float literals (#3327) * fix(spec): treat -0.0 and 0.0 as distinct float literals PrimitiveLiteral equality and ordering now follow Java's Float.compare and Double.compare: -0.0 is less than 0.0 and all NaNs are equal. This stops the partitioned writers, the record batch splitter and the delete file index from merging -0.0 and 0.0 identity partitions. Closes #3325 * Address review: fail closed in PrimitiveLiteral fallback, test IN - Handle AboveMax and BelowMin explicitly and return None when the fallback arm sees two values of the same variant, with a debug assertion, instead of calling them equal. - Cite the spec's partition equality rule and document the variant_index ordering invariant. - Stop pinning the cross-variant order in tests. - Test IN with signed zeros in the expression evaluator. --- crates/iceberg/public-api.txt | 5 +- .../arrow/record_batch_partition_splitter.rs | 75 +++++++++++++++- crates/iceberg/src/delete_file_index.rs | 32 +++++++ .../src/expr/visitors/expression_evaluator.rs | 70 +++++++++++++++ crates/iceberg/src/spec/values/primitive.rs | 80 ++++++++++++++++- crates/iceberg/src/spec/values/tests.rs | 59 +++++++++++++ .../writer/partitioning/clustered_writer.rs | 84 +++++++++++++++++- .../src/writer/partitioning/fanout_writer.rs | 87 ++++++++++++++++++- 8 files changed, 483 insertions(+), 9 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index bd55395346..8a34bbf7b5 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1693,16 +1693,15 @@ impl core::clone::Clone for iceberg::spec::PrimitiveLiteral pub fn iceberg::spec::PrimitiveLiteral::clone(&self) -> iceberg::spec::PrimitiveLiteral impl core::cmp::Eq for iceberg::spec::PrimitiveLiteral impl core::cmp::PartialEq for iceberg::spec::PrimitiveLiteral -pub fn iceberg::spec::PrimitiveLiteral::eq(&self, other: &iceberg::spec::PrimitiveLiteral) -> bool +pub fn iceberg::spec::PrimitiveLiteral::eq(&self, other: &Self) -> bool impl core::cmp::PartialOrd for iceberg::spec::PrimitiveLiteral -pub fn iceberg::spec::PrimitiveLiteral::partial_cmp(&self, other: &iceberg::spec::PrimitiveLiteral) -> core::option::Option +pub fn iceberg::spec::PrimitiveLiteral::partial_cmp(&self, other: &Self) -> core::option::Option impl core::convert::From for iceberg::spec::PrimitiveLiteral pub fn iceberg::spec::PrimitiveLiteral::from(value: iceberg::spec::Datum) -> Self impl core::fmt::Debug for iceberg::spec::PrimitiveLiteral pub fn iceberg::spec::PrimitiveLiteral::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::hash::Hash for iceberg::spec::PrimitiveLiteral pub fn iceberg::spec::PrimitiveLiteral::hash<__H: core::hash::Hasher>(&self, state: &mut __H) -impl core::marker::StructuralPartialEq for iceberg::spec::PrimitiveLiteral pub enum iceberg::spec::PrimitiveType pub iceberg::spec::PrimitiveType::Binary pub iceberg::spec::PrimitiveType::Boolean diff --git a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs index 0b679040c6..af3c99a979 100644 --- a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs +++ b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs @@ -209,7 +209,7 @@ impl RecordBatchPartitionSplitter { mod tests { use std::sync::Arc; - use arrow_array::{Int32Array, RecordBatch, StringArray}; + use arrow_array::{Float64Array, Int32Array, RecordBatch, StringArray}; use arrow_schema::DataType; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; @@ -474,4 +474,77 @@ mod tests { assert_eq!(ids, vec![3, 3]); assert_eq!(names, vec!["d", "f"]); } + + #[test] + fn test_record_batch_partition_split_signed_zero() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "id", + Type::Primitive(crate::spec::PrimitiveType::Int), + ) + .into(), + NestedField::required( + 2, + "d", + Type::Primitive(crate::spec::PrimitiveType::Double), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + PartitionSpecBuilder::new(schema.clone()) + .with_spec_id(1) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("d".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(), + ); + let partition_splitter = RecordBatchPartitionSplitter::try_new_with_computed_values( + schema.clone(), + partition_spec, + ) + .expect("Failed to create splitter"); + + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).unwrap()); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Float64Array::from(vec![-0.0, 0.0, -0.0])), + ]) + .expect("Failed to create RecordBatch"); + + let mut partitions: Vec<(Struct, Vec)> = partition_splitter + .split(&batch) + .expect("Failed to split RecordBatch") + .into_iter() + .map(|(key, batch)| { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + (key.data().clone(), ids) + }) + .collect(); + partitions.sort_by_key(|(_, ids)| ids[0]); + + // -0.0 and 0.0 are different partition values, as in iceberg-java. + assert_eq!(partitions, vec![ + (Struct::from_iter([Some(Literal::double(-0.0))]), vec![1, 3]), + (Struct::from_iter([Some(Literal::double(0.0))]), vec![2]), + ]); + } } diff --git a/crates/iceberg/src/delete_file_index.rs b/crates/iceberg/src/delete_file_index.rs index 8bae439410..01b4773f62 100644 --- a/crates/iceberg/src/delete_file_index.rs +++ b/crates/iceberg/src/delete_file_index.rs @@ -592,6 +592,38 @@ mod tests { assert!(actual_paths_to_apply_for_different_spec.is_empty()); } + #[test] + fn test_partition_scoped_deletes_distinguish_signed_zero() { + let neg_zero = Struct::from_iter([Some(Literal::double(-0.0))]); + let pos_zero = Struct::from_iter([Some(Literal::double(0.0))]); + let spec_id = 1; + + let delete_contexts: Vec = [ + build_partitioned_eq_delete(&neg_zero, spec_id), + build_partitioned_pos_delete(&neg_zero, spec_id), + ] + .iter() + .map(|delete| DeleteFileContext { + manifest_entry: build_added_manifest_entry(4, delete).into(), + partition_spec_id: spec_id, + }) + .collect(); + + let delete_file_index = PopulatedDeleteFileIndex::new(delete_contexts).unwrap(); + + let deletes_for_neg_zero = delete_file_index + .get_deletes_for_data_file(&build_partitioned_data_file(&neg_zero, spec_id), Some(0)) + .unwrap(); + assert_eq!(deletes_for_neg_zero.len(), 2); + + // -0.0 and 0.0 are different partitions, as in iceberg-java, so deletes written + // for one do not apply to data files in the other. + let deletes_for_pos_zero = delete_file_index + .get_deletes_for_data_file(&build_partitioned_data_file(&pos_zero, spec_id), Some(0)) + .unwrap(); + assert!(deletes_for_pos_zero.is_empty()); + } + #[test] fn test_pos_delete_with_referenced_data_file_applies_only_to_that_file() { let data_file_a = build_unpartitioned_data_file(); diff --git a/crates/iceberg/src/expr/visitors/expression_evaluator.rs b/crates/iceberg/src/expr/visitors/expression_evaluator.rs index 31ce94a99b..d9c2e1b571 100644 --- a/crates/iceberg/src/expr/visitors/expression_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/expression_evaluator.rs @@ -596,6 +596,76 @@ mod tests { Ok(()) } + #[test] + fn test_expr_eq_signed_zero() -> Result<()> { + let case_sensitive = true; + let (partition_spec, schema) = create_partition_spec(PrimitiveType::Float)?; + + let data_file = DataFile { + partition: Struct::from_iter([Some(Literal::float(-0.0_f32))]), + ..create_data_file_float() + }; + + // -0.0 and 0.0 are different values, as in iceberg-java's evaluator. + for (literal, expected) in [(0.0_f32, false), (-0.0_f32, true)] { + let predicate = Predicate::Binary(BinaryExpression::new( + PredicateOperator::Eq, + Reference::new("a"), + Datum::float(literal), + )) + .bind(schema.clone(), case_sensitive)?; + + let expression_evaluator = create_expression_evaluator( + partition_spec.clone(), + &schema, + &predicate, + case_sensitive, + )?; + + assert_eq!(expression_evaluator.eval(&data_file)?, expected); + } + + Ok(()) + } + + #[test] + fn test_expr_in_signed_zero() -> Result<()> { + let case_sensitive = true; + let (partition_spec, schema) = create_partition_spec(PrimitiveType::Float)?; + + let data_file = DataFile { + partition: Struct::from_iter([Some(Literal::float(-0.0_f32))]), + ..create_data_file_float() + }; + + // -0.0 and 0.0 are different values, so `IN` keeps both and matches only its own zero. + for (literals, expected) in [ + ([0.0_f32, 1.0_f32], false), + ([-0.0_f32, 1.0_f32], true), + ([0.0_f32, -0.0_f32], true), + ] { + let predicate = Predicate::Set(SetExpression::new( + PredicateOperator::In, + Reference::new("a"), + FnvHashSet::from_iter(literals.map(Datum::float)), + )) + .bind(schema.clone(), case_sensitive)?; + + assert!(matches!(&predicate, BoundPredicate::Set(expr) if expr.literals().len() == 2)); + + let expression_evaluator = create_expression_evaluator( + partition_spec.clone(), + &schema, + &predicate, + case_sensitive, + )?; + + assert_eq!(expression_evaluator.eval(&data_file)?, expected); + } + + Ok(()) + } + #[test] fn test_expr_greater_than_or_eq() -> Result<()> { let case_sensitive = true; diff --git a/crates/iceberg/src/spec/values/primitive.rs b/crates/iceberg/src/spec/values/primitive.rs index 43d5c48c54..b76978bd5f 100644 --- a/crates/iceberg/src/spec/values/primitive.rs +++ b/crates/iceberg/src/spec/values/primitive.rs @@ -17,10 +17,21 @@ //! Primitive literal types -use ordered_float::OrderedFloat; +use std::cmp::Ordering; + +use ordered_float::{FloatCore, OrderedFloat}; /// Values present in iceberg type -#[derive(Clone, Debug, PartialOrd, PartialEq, Hash, Eq)] +/// +/// `Float` and `Double` compare as Java's `Float.compare` and `Double.compare` do: `-0.0` is +/// less than `0.0`, and every NaN is the same value, greater than all others. iceberg-java +/// compares partition values this way, so `-0.0` and `0.0` are different partitions. The +/// [spec](https://iceberg.apache.org/spec/#scan-planning) states the same rule: floating point +/// partition values are equal if their IEEE 754 bit layouts are equal, with NaNs normalized. +// The derived `Hash` gives `-0.0` and `0.0` the same hash. That is coarser than `eq`, which is +// allowed: equal values still hash alike. +#[allow(clippy::derived_hash_with_manual_eq)] +#[derive(Clone, Debug, Hash, Eq)] pub enum PrimitiveLiteral { /// 0x00 for false, non-zero byte for true Boolean(bool), @@ -46,7 +57,72 @@ pub enum PrimitiveLiteral { BelowMin, } +impl PartialEq for PrimitiveLiteral { + fn eq(&self, other: &Self) -> bool { + self.partial_cmp(other) == Some(Ordering::Equal) + } +} + +impl PartialOrd for PrimitiveLiteral { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Boolean(a), Self::Boolean(b)) => a.partial_cmp(b), + (Self::Int(a), Self::Int(b)) => a.partial_cmp(b), + (Self::Long(a), Self::Long(b)) => a.partial_cmp(b), + (Self::Float(a), Self::Float(b)) => Some(float_cmp(a, b)), + (Self::Double(a), Self::Double(b)) => Some(float_cmp(a, b)), + (Self::String(a), Self::String(b)) => a.partial_cmp(b), + (Self::Binary(a), Self::Binary(b)) => a.partial_cmp(b), + (Self::Int128(a), Self::Int128(b)) => a.partial_cmp(b), + (Self::UInt128(a), Self::UInt128(b)) => a.partial_cmp(b), + (Self::AboveMax, Self::AboveMax) | (Self::BelowMin, Self::BelowMin) => { + Some(Ordering::Equal) + } + // Different variants order by declaration, as the derived impl did. A variant that + // lacks an arm above lands here against itself; fail closed instead of calling the + // two values equal. + _ => { + debug_assert_ne!(std::mem::discriminant(self), std::mem::discriminant(other)); + match self.variant_index().cmp(&other.variant_index()) { + Ordering::Equal => None, + ordering => Some(ordering), + } + } + } + } +} + +/// Compares floats as Java's `Float.compare` and `Double.compare` do. `OrderedFloat` already +/// treats every NaN as one value above all others; it only lacks `-0.0` before `0.0`. +fn float_cmp(a: &OrderedFloat, b: &OrderedFloat) -> Ordering { + a.cmp(b).then_with(|| { + if a.is_nan() { + Ordering::Equal + } else { + b.is_sign_negative().cmp(&a.is_sign_negative()) + } + }) +} + impl PrimitiveLiteral { + /// Must follow the declaration order of the variants: `partial_cmp` uses it to order + /// different variants the way the derived `PartialOrd` did. + fn variant_index(&self) -> u8 { + match self { + Self::Boolean(_) => 0, + Self::Int(_) => 1, + Self::Long(_) => 2, + Self::Float(_) => 3, + Self::Double(_) => 4, + Self::String(_) => 5, + Self::Binary(_) => 6, + Self::Int128(_) => 7, + Self::UInt128(_) => 8, + Self::AboveMax => 9, + Self::BelowMin => 10, + } + } + /// Returns true if the Literal represents a primitive type /// that can be a NaN, and that it's value is NaN pub fn is_nan(&self) -> bool { diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index c756bc3f8e..61f83efee1 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -17,6 +17,8 @@ //! Tests for Iceberg value types +use std::collections::HashSet; + use apache_avro::to_value; use apache_avro::types::Value; use ordered_float::OrderedFloat; @@ -1534,6 +1536,63 @@ fn test_negative_zero_less_than_positive_zero() { } } +#[test] +fn test_signed_zero_literals_are_distinct() { + for (neg_zero, pos_zero) in [ + ( + PrimitiveLiteral::Float(OrderedFloat(-0.0)), + PrimitiveLiteral::Float(OrderedFloat(0.0)), + ), + ( + PrimitiveLiteral::Double(OrderedFloat(-0.0)), + PrimitiveLiteral::Double(OrderedFloat(0.0)), + ), + ] { + assert_ne!(neg_zero, pos_zero); + assert_eq!( + neg_zero.partial_cmp(&pos_zero), + Some(std::cmp::Ordering::Less) + ); + assert_eq!(HashSet::from([neg_zero, pos_zero]).len(), 2); + } + + assert_ne!(Datum::float(-0.0_f32), Datum::float(0.0_f32)); + assert_ne!(Datum::double(-0.0), Datum::double(0.0)); + + // A partition tuple is a `Struct`, which the partitioned writers use as a map key. + let partitions = HashSet::from([ + Struct::from_iter([Some(Literal::double(-0.0))]), + Struct::from_iter([Some(Literal::double(0.0))]), + ]); + assert_eq!(partitions.len(), 2); +} + +#[test] +fn test_nan_literals_are_equal() { + for (nan, other_nan) in [ + ( + PrimitiveLiteral::Float(OrderedFloat(f32::NAN)), + PrimitiveLiteral::Float(OrderedFloat(-f32::NAN)), + ), + ( + PrimitiveLiteral::Double(OrderedFloat(f64::NAN)), + PrimitiveLiteral::Double(OrderedFloat(-f64::NAN)), + ), + ] { + assert_eq!(nan, other_nan); + assert_eq!(nan.partial_cmp(&other_nan), Some(std::cmp::Ordering::Equal)); + assert_eq!(HashSet::from([nan, other_nan]).len(), 1); + } +} + +#[test] +fn test_primitive_literal_eq_across_variants() { + assert_eq!(PrimitiveLiteral::AboveMax, PrimitiveLiteral::AboveMax); + assert_eq!(PrimitiveLiteral::BelowMin, PrimitiveLiteral::BelowMin); + assert_ne!(PrimitiveLiteral::AboveMax, PrimitiveLiteral::BelowMin); + assert_ne!(PrimitiveLiteral::Int(1), PrimitiveLiteral::Long(1)); +} + /// Test Date deserialization from JSON as number (days since epoch). /// /// This reproduces the scenario from Iceberg Java's TestAddFilesProcedure where: diff --git a/crates/iceberg/src/writer/partitioning/clustered_writer.rs b/crates/iceberg/src/writer/partitioning/clustered_writer.rs index 1fd9738c4d..21b52bd813 100644 --- a/crates/iceberg/src/writer/partitioning/clustered_writer.rs +++ b/crates/iceberg/src/writer/partitioning/clustered_writer.rs @@ -145,13 +145,14 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use arrow_array::{Int32Array, RecordBatch, StringArray}; + use arrow_array::{Float64Array, Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::file::properties::WriterProperties; use tempfile::TempDir; use super::*; + use crate::arrow::schema_to_arrow_schema; use crate::io::FileIO; use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Type}; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; @@ -508,4 +509,85 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_clustered_writer_signed_zero_partitions() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "d", Type::Primitive(PrimitiveType::Double)).into(), + ]) + .build()?, + ); + let partition_spec = crate::spec::PartitionSpec::builder(schema.clone()) + .add_partition_field("d", "d", crate::spec::Transform::Identity)? + .build()?; + + let partition_value = |d: f64| Struct::from_iter([Some(crate::spec::Literal::double(d))]); + let partition_key = + |d: f64| PartitionKey::new(partition_spec.clone(), schema.clone(), partition_value(d)); + + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema)?); + let batch = |id: i32, d: f64| { + RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![id])), + Arc::new(Float64Array::from(vec![d])), + ]) + }; + + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + + // -0.0 and 0.0 are different partition values, as in iceberg-java, so 0.0 right + // after -0.0 starts a new data file. + let mut writer = + ClusteredWriter::new(DataFileWriterBuilder::new(rolling_writer_builder.clone())); + writer.write(partition_key(-0.0), batch(1, -0.0)?).await?; + writer.write(partition_key(0.0), batch(2, 0.0)?).await?; + let partitions_written: Vec = writer + .close() + .await? + .into_iter() + .map(|data_file| data_file.partition) + .collect(); + assert_eq!(partitions_written, vec![ + partition_value(-0.0), + partition_value(0.0) + ]); + + // 0.0 is not the closed -0.0 partition, so this input is still sorted. + let mut writer = ClusteredWriter::new(DataFileWriterBuilder::new(rolling_writer_builder)); + writer.write(partition_key(-0.0), batch(1, -0.0)?).await?; + writer.write(partition_key(1.0), batch(3, 1.0)?).await?; + writer.write(partition_key(0.0), batch(2, 0.0)?).await?; + let partitions_written: Vec = writer + .close() + .await? + .into_iter() + .map(|data_file| data_file.partition) + .collect(); + assert_eq!(partitions_written, vec![ + partition_value(-0.0), + partition_value(1.0), + partition_value(0.0) + ]); + + Ok(()) + } } diff --git a/crates/iceberg/src/writer/partitioning/fanout_writer.rs b/crates/iceberg/src/writer/partitioning/fanout_writer.rs index 5348fea157..2cdd865375 100644 --- a/crates/iceberg/src/writer/partitioning/fanout_writer.rs +++ b/crates/iceberg/src/writer/partitioning/fanout_writer.rs @@ -119,17 +119,18 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use arrow_array::{Int32Array, RecordBatch, StringArray}; + use arrow_array::{Float64Array, Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::file::properties::WriterProperties; use tempfile::TempDir; use super::*; + use crate::arrow::schema_to_arrow_schema; use crate::io::FileIO; use crate::spec::{ DataFileFormat, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Struct, - Type, + Transform, Type, }; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; use crate::writer::file_writer::ParquetWriterBuilder; @@ -381,4 +382,86 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_fanout_writer_signed_zero_partitions() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "d", Type::Primitive(PrimitiveType::Double)).into(), + ]) + .build()?, + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("d", "d", Transform::Identity)? + .build()?; + + let partition_value_neg_zero = Struct::from_iter([Some(Literal::double(-0.0))]); + let partition_key_neg_zero = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_neg_zero.clone(), + ); + + let partition_value_pos_zero = Struct::from_iter([Some(Literal::double(0.0))]); + let partition_key_pos_zero = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_pos_zero.clone(), + ); + + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + let mut writer = FanoutWriter::new(data_file_writer_builder); + + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema)?); + let batch_neg_zero = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(Float64Array::from(vec![-0.0])), + ])?; + let batch_pos_zero = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![2])), + Arc::new(Float64Array::from(vec![0.0])), + ])?; + + writer.write(partition_key_neg_zero, batch_neg_zero).await?; + writer.write(partition_key_pos_zero, batch_pos_zero).await?; + + let data_files = writer.close().await?; + + // -0.0 and 0.0 are different partition values, as in iceberg-java, so each row + // goes to its own data file. + assert_eq!(data_files.len(), 2); + for data_file in &data_files { + assert_eq!(data_file.record_count, 1); + } + let partitions_found: std::collections::HashSet = data_files + .iter() + .map(|data_file| data_file.partition.clone()) + .collect(); + assert_eq!( + partitions_found, + std::collections::HashSet::from([partition_value_neg_zero, partition_value_pos_zero]) + ); + + Ok(()) + } } From e8ec7f8403554496db976700eb993459e0ab7f9b Mon Sep 17 00:00:00 2001 From: Danny C Jones Date: Fri, 9 Oct 2026 22:31:27 +0000 Subject: [PATCH 48/63] chore(release): port the dependency license check from shell to Python (#3379) This change migrates from the old shell script for cargo-deny to a new Python script. This new script will contain logic for dealing with dependencies and their licenses. This commit simply translates the existing logic. In later changes, this script will be extended with: - Clarifications for existing dependencies where licenses are misreported, with additional verifications added in the script itself - Logic for generating the attribution documentation for the pyiceberg-core package. --- .github/workflows/ci.yml | 6 +- Makefile | 2 +- dev/release/README.md | 4 +- dev/release/create_rc.sh | 3 +- dev/release/dependencies.sh | 148 ------------------------------------ dev/release/licenses.py | 101 ++++++++++++++++++++++++ website/src/release.md | 2 +- 7 files changed, 113 insertions(+), 153 deletions(-) delete mode 100755 dev/release/dependencies.sh create mode 100755 dev/release/licenses.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c57b4799c..9369a91e88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,10 @@ jobs: with: tool: taplo@0.9.3,cargo-machete@0.7.0,cargo-deny@0.19.9 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + - name: Check License Header uses: apache/skywalking-eyes/header@a196742f472feaffafea537ce5a2a4c3c53a8de4 # v0.9.0 @@ -80,7 +84,7 @@ jobs: run: make check-license-notice - name: Check dependency licenses - run: dev/release/dependencies.sh check + run: python3 ./dev/release/licenses.py dependencies check # Trusted publishing cannot create a crate (https://crates.io/docs/trusted-publishing), # so a new publishable crate must be reserved on crates.io before it is merged. diff --git a/Makefile b/Makefile index 8733abbab5..c596fc4bf7 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ install-cargo-deny: cargo install --locked cargo-deny@0.19.9 check-dependency-licenses: install-cargo-deny - bash ./dev/release/dependencies.sh check + python3 ./dev/release/licenses.py dependencies check check: check-fmt check-clippy check-toml cargo-machete check-license-notice check-dependency-licenses diff --git a/dev/release/README.md b/dev/release/README.md index cf10a3aa3f..4a9cecd8f8 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -137,5 +137,7 @@ git push origin "v0.9.1" Verify every dependency's license is one the project allows: ```shell -dev/release/dependencies.sh check +python3 dev/release/licenses.py dependencies check ``` + +CI also runs this check on every pull request. diff --git a/dev/release/create_rc.sh b/dev/release/create_rc.sh index ae3a5207ee..99dd37565b 100755 --- a/dev/release/create_rc.sh +++ b/dev/release/create_rc.sh @@ -307,7 +307,8 @@ check_rc_tag_available() { } check_dependency_licenses() { - "${SCRIPT_DIR}/dependencies.sh" check + require_command python3 + python3 "${SCRIPT_DIR}/licenses.py" dependencies check } # Packages and builds every crate as `cargo publish` would, without uploading, diff --git a/dev/release/dependencies.sh b/dev/release/dependencies.sh deleted file mode 100755 index 36246a1cdb..0000000000 --- a/dev/release/dependencies.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -Eeuo pipefail - -# bash 3.2 (macOS default) does not run the ERR trap for a failing subshell: -# the script still exits on failure but does not report which step failed. -if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then - echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 -fi - -CURRENT_STEP="" - -on_error() { - local status=$? - if [ -n "${CURRENT_STEP}" ]; then - echo "FAILED: ${CURRENT_STEP}" >&2 - else - echo "FAILED" >&2 - fi - exit "${status}" -} - -trap on_error ERR - -usage() { - cat < - -Commands: - check - Run cargo-deny license validation once at the root workspace. - Default command: none; this argument is required. - -Options: - -h, --help - Show this help message. - -Examples: - $0 check -USAGE -} - -show_help_if_requested() { - while [ "$#" -gt 0 ]; do - case "$1" in - -h | --help) - usage - exit 0 - ;; - esac - shift - done -} - -start_step() { - CURRENT_STEP="$1" - echo "==> ${CURRENT_STEP}" -} - -finish_step() { - echo "OK: ${CURRENT_STEP}" - CURRENT_STEP="" -} - -run_step() { - local step="$1" - shift - start_step "${step}" - "$@" - finish_step -} - -require_command() { - local command_name="$1" - if ! command -v "${command_name}" >/dev/null 2>&1; then - echo "This script requires '${command_name}', but it is not installed." >&2 - return 1 - fi -} - -require_cargo_deny() { - require_command cargo - if ! cargo deny --version >/dev/null 2>&1; then - echo "This script requires 'cargo-deny' for dependency license checks." >&2 - echo "Install it with: cargo install --locked cargo-deny" >&2 - return 1 - fi -} - -validate_args() { - if [ "$#" -ne 1 ]; then - usage - return 1 - fi - - case "$1" in - check) ;; - *) - usage - return 1 - ;; - esac -} - -check_deps_for_dir() { - local cargo_dir="$1" - require_cargo_deny - ( - trap - ERR - cd "${cargo_dir}" - cargo deny check license - ) -} - -check_deps() { - run_step "Check dependency licenses in ${REPO_ROOT}" check_deps_for_dir "${REPO_ROOT}" -} - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -COMMAND="${1:-}" - -show_help_if_requested "$@" -run_step "Validate dependency command arguments" validate_args "$@" - -case "${COMMAND}" in - check) - check_deps - ;; -esac diff --git a/dev/release/licenses.py b/dev/release/licenses.py new file mode 100755 index 0000000000..2fc7449878 --- /dev/null +++ b/dev/release/licenses.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +CLI for assisting maintenance of the project's license checks. +""" + +import argparse +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +class Failure(Exception): + """A user-facing error.""" + + +def step(message: str) -> None: + print(f"==> {message}", flush=True) + + +def ok(message: str) -> None: + print(f"OK: {message}", flush=True) + + +def require_cargo_deny() -> None: + """Assert cargo-deny is installed.""" + try: + subprocess.run( + ["cargo", "deny", "--version"], + capture_output=True, + check=True, + ) + except FileNotFoundError: + raise Failure("This script requires 'cargo', but it is not installed.") + except subprocess.CalledProcessError: + raise Failure( + "This script requires 'cargo-deny' for dependency license checks.\n" + "Install it with: make install-cargo-deny" + ) + + +def command_check_dependencies(_args: argparse.Namespace) -> None: + step("Check dependency licenses") + require_cargo_deny() + + result = subprocess.run( + ["cargo", "deny", "check", "license"], + cwd=REPO_ROOT, + check=False, + ) + if result.returncode != 0: + raise Failure("cargo-deny rejected the dependency licenses.") + + ok("Check dependency licenses") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + cmd_groups = parser.add_subparsers(dest="group", required=True) + + dependencies = cmd_groups.add_parser( + "dependencies", + help="license policy for the Rust dependency graph, enforced through deny.toml", + ) + dependencies_commands = dependencies.add_subparsers(dest="command", required=True) + + check = dependencies_commands.add_parser( + "check", + help="assert every dependency's license is one deny.toml allows", + ) + check.set_defaults(func=command_check_dependencies) + + args = parser.parse_args() + try: + args.func(args) + except Failure as failure: + print(str(failure), file=sys.stderr) + print(f"FAILED: {args.group} {args.command}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/website/src/release.md b/website/src/release.md index 03fb665515..a519fce561 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -175,7 +175,7 @@ The changelog should reflect a summary of each commit in the new release. Run the following command to verify the licenses meet the project's policy. ```shell -dev/release/dependencies.sh check +python3 dev/release/licenses.py dependencies check ``` #### Open pull request From b8467510f8da71b8f180affdef7b1a2d3dd4cfa9 Mon Sep 17 00:00:00 2001 From: Jiajia Li Date: Fri, 9 Oct 2026 23:17:10 +0000 Subject: [PATCH 49/63] feat(rest): add a SigV4 request signer (#3082) * feat(rest): add a SigV4 request signer * fix(rest): mute aws-sigv4 tracing and tighten payload hashing The crate traces the request it signs, and its redaction list covers `authorization` but not the `Original-` copy we make, so a delegate's bearer token could reach trace logs. Also reject non-UTF-8 headers rather than leave them unsigned, and hash a present-but-empty body as Java does. * refactor(rest): take credentials per call, as Java's signer does Java holds one `Aws4Signer` across sessions and resolves credentials from an `AwsCredentialsProvider` per request, so the signer itself carries no credential state. Follow that: drop `AwsCredentials`, take the AWS crate's `Credentials` as a `sign` argument, and leave the provider to the auth session. Also sign `HttpRequest` rather than the concrete request type, name the amz headers, split `convert_headers`/`update_request_headers` after their Java counterparts, collapse the one-file `sigv4` module, and put the AWS dependencies behind a `sigv4` feature. * docs(rest): trim the SigV4 signer comments * refactor(rest): break sign_at into named steps * docs(rest): state the redirect and log-always limits of the signer * fix(rest): keep relocation headers out of the signature, and the tracing flag untouched * docs(rest): name the log bridge that stays unmuted * docs(rest): log-always also bypasses the muted subscriber * fix(rest): return an error instead of unwrapping the content hash header * refactor(rest): make PayloadHashMode non_exhaustive and SigV4Signer::new take impl Into * docs(rest): note the base64 header quirk and credential freshness * docs(rest): keep the plus note next to the rewrite it explains * fix(rest): spell out the headers Java leaves unsigned * test(rest): check signatures against Iceberg Java * test(rest): pin the signing tests to a fixed time * test(rest): check standard mode against the AWS test suite * test(rest): drop the helper-only AWS vector and the hmac dev-dependency * fix(rest): decide the tracing mute with public tracing API * docs(rest): tighten the payload hash and credential notes * test(rest): bound the live-clock stamp by readings around the call * chore(rest): update the public API for the signer changes * docs(rest): simplify the SigV4 signer comments * refactor(rest): build SigV4Signer with a typed builder * fix(rest): propagate a failed userinfo strip instead of signing the URL as is * docs(rest): name the flag the tracing mute is gated to avoid * docs(rest): say that as_bytes merges an absent body with an empty one * chore(rest): update the public API for the signer builder * fix(rest): validate requests before signing * fix(rest): address SigV4 review findings --- Cargo.lock | 4 + Cargo.toml | 6 + crates/catalog/rest/Cargo.toml | 9 + crates/catalog/rest/public-api.txt | 22 + crates/catalog/rest/src/auth/mod.rs | 6 + crates/catalog/rest/src/auth/sigv4.rs | 1225 +++++++++++++++++++++++++ crates/catalog/rest/src/request.rs | 14 + 7 files changed, 1286 insertions(+) create mode 100644 crates/catalog/rest/src/auth/sigv4.rs diff --git a/Cargo.lock b/Cargo.lock index 75694c3875..f3694ad2e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2843,6 +2843,9 @@ name = "iceberg-catalog-rest" version = "0.10.1" dependencies = [ "async-trait", + "aws-credential-types", + "aws-sigv4", + "base64 0.22.1", "bytes", "chrono", "futures", @@ -2855,6 +2858,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "sha2 0.10.9", "tokio", "tracing", "typed-builder", diff --git a/Cargo.toml b/Cargo.toml index 24f8819949..bb9770275a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ arrow-string = "59.2" as-any = "0.3.2" async-trait = "0.1.89" aws-config = "1.8.7" +aws-credential-types = "1.2" aws-sdk-glue = { version = "1.85", default-features = false, features = [ "default-https-client", "rt-tokio", @@ -68,6 +69,10 @@ aws-sdk-s3tables = { version = "1.28", default-features = false, features = [ "default-https-client", "rt-tokio", ] } +aws-sigv4 = { version = "1.4", default-features = false, features = [ + "sign-http", + "http1", +] } backon = "1.5.1" base64 = "0.22.1" bimap = "0.6" @@ -124,6 +129,7 @@ serde_derive = "1.0.219" serde_json = "1.0.142" serde_repr = "0.1.16" serde_with = "3.4" +sha2 = "0.10" sqlx = { version = "0.8.1", default-features = false } strum = "0.27.2" syn = "2" diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index 8dc9a86d7f..0a8cf6b6e3 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -29,8 +29,16 @@ keywords = ["iceberg", "rest", "catalog"] license = { workspace = true } repository = { workspace = true } +[features] +default = [] +# SigV4 request signing. Off by default to keep the AWS crates out. +sigv4 = ["dep:aws-credential-types", "dep:aws-sigv4", "dep:base64", "dep:sha2"] + [dependencies] async-trait = { workspace = true } +aws-credential-types = { workspace = true, optional = true } +aws-sigv4 = { workspace = true, optional = true } +base64 = { workspace = true, optional = true } chrono = { workspace = true } http = { workspace = true } iceberg = { workspace = true } @@ -39,6 +47,7 @@ reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true, optional = true } tokio = { workspace = true } tracing = { workspace = true } typed-builder = { workspace = true } diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index 47393b5738..ddfafb11d6 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -1,4 +1,5 @@ pub mod iceberg_catalog_rest +pub use iceberg_catalog_rest::Credentials pub enum iceberg_catalog_rest::HttpRequestBody<'a> pub iceberg_catalog_rest::HttpRequestBody::Buffered(&'a [u8]) pub iceberg_catalog_rest::HttpRequestBody::Empty @@ -14,6 +15,18 @@ impl<'a> core::fmt::Debug for iceberg_catalog_rest::HttpRequestBody<'a> pub fn iceberg_catalog_rest::HttpRequestBody<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl<'a> core::marker::Copy for iceberg_catalog_rest::HttpRequestBody<'a> impl<'a> core::marker::StructuralPartialEq for iceberg_catalog_rest::HttpRequestBody<'a> +#[non_exhaustive] pub enum iceberg_catalog_rest::PayloadHashMode +pub iceberg_catalog_rest::PayloadHashMode::IcebergRest +pub iceberg_catalog_rest::PayloadHashMode::StandardAws +impl core::clone::Clone for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::clone(&self) -> iceberg_catalog_rest::PayloadHashMode +impl core::cmp::Eq for iceberg_catalog_rest::PayloadHashMode +impl core::cmp::PartialEq for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::eq(&self, other: &iceberg_catalog_rest::PayloadHashMode) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg_catalog_rest::PayloadHashMode +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::PayloadHashMode pub struct iceberg_catalog_rest::CommitTableRequest pub iceberg_catalog_rest::CommitTableRequest::identifier: core::option::Option pub iceberg_catalog_rest::CommitTableRequest::requirements: alloc::vec::Vec @@ -331,6 +344,15 @@ impl core::default::Default for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::default() -> Self impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_catalog_rest::SigV4Signer +impl iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut iceberg_catalog_rest::HttpRequest, credentials: &aws_credential_types::credentials_impl::Credentials) -> iceberg::error::Result<()> +impl core::clone::Clone for iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::clone(&self) -> iceberg_catalog_rest::SigV4Signer +impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::builder() -> SigV4SignerBuilder<((), (), ())> pub struct iceberg_catalog_rest::StorageCredential pub iceberg_catalog_rest::StorageCredential::config: std::collections::hash::map::HashMap pub iceberg_catalog_rest::StorageCredential::prefix: alloc::string::String diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs index b4c0cf0be2..a3aab4df90 100644 --- a/crates/catalog/rest/src/auth/mod.rs +++ b/crates/catalog/rest/src/auth/mod.rs @@ -19,14 +19,20 @@ //! `AuthManager`/`AuthSession` API. mod oauth2; +#[cfg(feature = "sigv4")] +mod sigv4; use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; use async_trait::async_trait; +#[cfg(feature = "sigv4")] +pub use aws_credential_types::Credentials; use iceberg::{Result, SessionContext}; pub use oauth2::OAuth2Manager; +#[cfg(feature = "sigv4")] +pub use sigv4::{PayloadHashMode, SigV4Signer}; use crate::client::HttpClient; use crate::request::HttpRequest; diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs new file mode 100644 index 0000000000..b7b4ae5bf1 --- /dev/null +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -0,0 +1,1225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for the REST catalog. + +use chrono::{DateTime, Utc}; +use iceberg::{Error, ErrorKind, Result}; +use sha2::{Digest, Sha256}; +use typed_builder::TypedBuilder; + +use super::Credentials; + +/// How the payload hash is encoded in the `x-amz-content-sha256` header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum PayloadHashMode { + /// Iceberg Java's style: a base64 header when there is a body, hex when + /// there is none, and hex in the canonical request. The base64 comes from + /// how Java configures the AWS SDK, not from SigV4, so a verifier that + /// trusts the header instead of hashing the body rejects it. + IcebergRest, + /// Standard SigV4: hex everywhere. + StandardAws, +} + +fn encode_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn base64_encode(bytes: &[u8]) -> String { + base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +/// The content header and canonical payload hash, computed from one digest. +fn payload_hashes(body: Option<&[u8]>, mode: PayloadHashMode) -> (String, String) { + let digest = Sha256::digest(body.unwrap_or_default()); + let hex = encode_hex(&digest); + let header = match mode { + PayloadHashMode::IcebergRest if body.is_some() => base64_encode(&digest), + _ => hex.clone(), + }; + (header, hex) +} + +/// Signs REST catalog requests the way Iceberg Java's `RESTSigV4AuthSession` +/// does. Carries no credentials, so one signer serves every session. +/// +/// Built with [`SigV4Signer::builder`]. +#[derive(Clone, Debug, TypedBuilder)] +pub struct SigV4Signer { + /// The signing region, e.g. `us-east-1`. + #[builder(setter(into))] + region: String, + /// The signing name, e.g. `execute-api`. + #[builder(setter(into))] + service: String, + /// How the payload hash is encoded. + mode: PayloadHashMode, +} + +impl SigV4Signer { + /// Signs `request` in place. An existing `Authorization` moves to + /// `Original-Authorization`, and a caller's `x-amz-date`, + /// `x-amz-content-sha256` or `x-amz-security-token` that the signer + /// overwrites moves to `Original-` (Java keeps a caller's content + /// hash when there is a body). Userinfo leaves the URL, and a `+` in the + /// query becomes `%20`, so write a literal plus as `%2B`. + /// + /// Uses `credentials` as given and never refreshes them: resolve temporary + /// ones from their provider before each call. + /// + /// On error, including for a streaming body or a non-UTF-8 header, the + /// request is left unchanged. + /// + /// Send the result through a client that does not follow redirects: a + /// redirect replays the signature, and across hosts reqwest drops + /// `Authorization` but keeps `Original-Authorization`. + /// + /// `aws_sigv4` traces requests without redacting `Original-Authorization`. + /// A `tracing` subscriber is muted for the call, but the `log` bridge (no + /// subscriber, or `log-always`) still forwards those events, so keep + /// `aws_sigv4` below trace level there. + pub fn sign(&self, request: &mut crate::HttpRequest, credentials: &Credentials) -> Result<()> { + self.sign_at(request, credentials, Utc::now()) + } + + fn sign_at( + &self, + request: &mut crate::HttpRequest, + credentials: &Credentials, + now: DateTime, + ) -> Result<()> { + let headers = request.headers().clone(); + let url = request.url().clone(); + let signed = self.sign_in_place(request, credentials, now); + if signed.is_err() { + *request.headers_mut() = headers; + *request.url_mut() = url; + } + signed + } + + fn sign_in_place( + &self, + request: &mut crate::HttpRequest, + credentials: &Credentials, + now: DateTime, + ) -> Result<()> { + use aws_sigv4::http_request::{SignableBody, SignableRequest, sign}; + use aws_sigv4::sign::v4; + use tracing::dispatcher::Dispatch; + use tracing::level_filters::LevelFilter; + + // Before the content hash is displaced, which takes it out of signing. + signable_headers(request).try_for_each(|header| header.map(|_| ()))?; + let (content_header, payload_hash) = payload_hashes(signable_body(request)?, self.mode); + + convert_headers(request); + + // Relocated after signing, so the `Original-` copy is not signed. + let displaced_content_hash: Vec<_> = request + .headers() + .get_all(CONTENT_SHA256) + .iter() + .filter(|v| v.as_bytes() != content_header.as_bytes()) + .cloned() + .collect(); + let content_value = content_header.parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid x-amz-content-sha256 value").with_source(e) + })?; + request.headers_mut().insert(CONTENT_SHA256, content_value); + + rewrite_url_for_signing(request)?; + + let identity = credentials.clone().into(); + let params = v4::SigningParams::builder() + .identity(&identity) + .region(&self.region) + .name(&self.service) + .time(now.into()) + .settings(signing_settings()) + .build() + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "failed to build SigV4 params").with_source(e) + })? + .into(); + + let headers = signable_headers(request).collect::>>()?; + let signable = SignableRequest::new( + request.method().as_str(), + request.url_str(), + headers.into_iter(), + SignableBody::Precomputed(payload_hash), + ) + .map_err(|e| { + Error::new(ErrorKind::DataInvalid, "request is not signable").with_source(e) + })?; + + // `aws_sigv4` traces the request, `Original-Authorization` included. + // Mute it only when a subscriber could record that (the max level is + // `OFF` until one is registered): `with_default` also sets + // tracing-core's `EXISTS` flag, which nothing clears, and with the + // `log` feature tracing then stops forwarding events to `log` for good. + let signed = if LevelFilter::current() == LevelFilter::TRACE { + tracing::dispatcher::with_default(&Dispatch::none(), || sign(signable, ¶ms)) + } else { + sign(signable, ¶ms) + }; + let (instructions, _signature) = signed + .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 signing failed").with_source(e))? + .into_parts(); + + update_request_headers(request, instructions, displaced_content_hash) + } +} + +/// The body to sign; as in Java, an absent body and an empty one differ. +fn signable_body(request: &crate::HttpRequest) -> Result> { + match request.body() { + crate::HttpRequestBody::Empty => Ok(None), + crate::HttpRequestBody::Buffered(bytes) => Ok(Some(bytes)), + crate::HttpRequestBody::Streaming => Err(Error::new( + ErrorKind::FeatureUnsupported, + "cannot sign a streaming request body", + )), + } +} + +/// The headers to sign. A non-UTF-8 one is an error: skipping it would send +/// it unsigned. +fn signable_headers(request: &crate::HttpRequest) -> impl Iterator> { + request.headers().iter().map(|(n, v)| { + let v = std::str::from_utf8(v.as_bytes()).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("cannot sign non-UTF-8 header value for `{n}`"), + ) + .with_source(e) + })?; + Ok((n.as_str(), v)) + }) +} + +/// Drops userinfo, which the wire `Host` never carries, and rewrites `+` in the +/// query as `%20`: a space to AWS and Java either way, but unambiguous to any +/// verifier. +fn rewrite_url_for_signing(request: &mut crate::HttpRequest) -> Result<()> { + if !request.url().username().is_empty() || request.url().password().is_some() { + let url = request.url_mut(); + url.set_username("") + .and_then(|()| url.set_password(None)) + .map_err(|()| { + Error::new( + ErrorKind::DataInvalid, + "cannot strip userinfo from the request URL", + ) + })?; + } + if let Some(query) = request.url().query().filter(|q| q.contains('+')) { + let unambiguous = query.replace('+', "%20"); + request.url_mut().set_query(Some(&unambiguous)); + } + Ok(()) +} + +/// `Aws4Signer`'s settings: normalized double-encoded path, and Java's ignore +/// list. +fn signing_settings() -> aws_sigv4::http_request::SigningSettings { + use aws_sigv4::http_request::{ + PayloadChecksumKind, PercentEncodingMode, SigningSettings, UriPathNormalizationMode, + }; + + let mut settings = SigningSettings::default(); + settings.percent_encoding_mode = PercentEncodingMode::Double; + settings.uri_path_normalization_mode = UriPathNormalizationMode::Enabled; + // We set and relocate the content hash ourselves, outside the instructions. + settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; + let mut excluded = settings.excluded_headers.take().unwrap_or_default(); + excluded.extend([ + // Java's list, spelled out rather than left to the crate's defaults. + "connection".into(), + "expect".into(), + "transfer-encoding".into(), + "user-agent".into(), + "x-amzn-trace-id".into(), + // Not Java's, but proxies append to it too. + "x-forwarded-for".into(), + // Relocation appends to these after signing. + "original-x-amz-date".into(), + "original-x-amz-content-sha256".into(), + "original-x-amz-security-token".into(), + ]); + settings.excluded_headers = Some(excluded); + settings +} + +/// Java's `convertHeaders`: moves `Authorization` aside before signing, so the +/// moved copy is signed. +fn convert_headers(request: &mut crate::HttpRequest) { + let displaced: Vec<_> = request + .headers() + .get_all(reqwest::header::AUTHORIZATION) + .iter() + .cloned() + .collect(); + if displaced.is_empty() { + return; + } + request.headers_mut().remove(reqwest::header::AUTHORIZATION); + for mut value in displaced { + value.set_sensitive(true); + request.headers_mut().append(RELOCATED_AUTHORIZATION, value); + } +} + +/// Java's `updateRequestHeaders`: installs the signed headers, moving +/// conflicting caller values aside. +fn update_request_headers( + request: &mut crate::HttpRequest, + instructions: aws_sigv4::http_request::SigningInstructions, + displaced_content_hash: Vec, +) -> Result<()> { + let (signed_headers, _params) = instructions.into_parts(); + let h = request.headers_mut(); + for mut value in displaced_content_hash { + // The original may carry a credential. + value.set_sensitive(true); + h.append(RELOCATED_CONTENT_SHA256, value); + } + for header in signed_headers { + let name: reqwest::header::HeaderName = header.name().parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid signed header name").with_source(e) + })?; + if let Some(relocated) = relocated_name(name.as_str()) { + relocate_conflicting(h, name.as_str(), header.value(), relocated); + } + let mut value: reqwest::header::HeaderValue = header.value().parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid signed header value").with_source(e) + })?; + if name == reqwest::header::AUTHORIZATION || name == SECURITY_TOKEN { + value.set_sensitive(true); + } + h.insert(name, value); + } + Ok(()) +} + +const CONTENT_SHA256: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-content-sha256"); +const AMZ_DATE: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-date"); +const SECURITY_TOKEN: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-security-token"); + +const RELOCATED_AUTHORIZATION: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-authorization"); +const RELOCATED_AMZ_DATE: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-date"); +const RELOCATED_CONTENT_SHA256: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-content-sha256"); +const RELOCATED_SECURITY_TOKEN: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-security-token"); + +/// The `Original-` counterpart of a header the signer generates. +fn relocated_name(name: &str) -> Option { + match name { + n if n == AMZ_DATE => Some(RELOCATED_AMZ_DATE), + n if n == SECURITY_TOKEN => Some(RELOCATED_SECURITY_TOKEN), + _ => None, + } +} + +/// Moves `name`'s values that differ from `signed` to `relocated`. +fn relocate_conflicting( + headers: &mut reqwest::header::HeaderMap, + name: &str, + signed: &str, + relocated: reqwest::header::HeaderName, +) { + let conflicting: Vec<_> = headers + .get_all(name) + .iter() + .filter(|value| value.as_bytes() != signed.as_bytes()) + .cloned() + .collect(); + for mut value in conflicting { + // It may carry a credential, e.g. a session token. + value.set_sensitive(true); + headers.append(relocated.clone(), value); + } +} + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + + use super::*; + use crate::HttpRequest; + + const EMPTY_HEX: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + #[test] + fn signing_rewrites_an_ambiguous_plus_out_of_the_query() { + // reqwest writes a space as `+`; signing makes it `%20`. + let mut request = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/namespaces") + .query(&[("parent", "my ns")]) + .build() + .unwrap(), + ); + assert!(request.url().query().unwrap().contains("my+ns")); + + let signer = test_signer(PayloadHashMode::StandardAws); + let now = test_time(); + signer + .sign_at(&mut request, &test_credentials(), now) + .unwrap(); + + let query = request.url().query().unwrap(); + assert!(!query.contains('+'), "{query}"); + assert!(query.contains("my%20ns"), "{query}"); + assert_signature_is( + &request, + "b7bb5a323a1ce0ace18454171084deef2dac44c933c3949771fc70179d3cce2b", + ); + } + + #[test] + fn payload_hashes_iceberg_mode() { + let v = payload_hashes(Some(b"hello"), PayloadHashMode::IcebergRest).0; + assert_eq!(v, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="); + let e = payload_hashes(None, PayloadHashMode::IcebergRest).0; + assert_eq!(e, EMPTY_HEX); + } + + /// As in Java, an empty body is hashed, unlike an absent one. + #[test] + fn signing_separates_an_empty_body_from_an_absent_one() { + let signer = test_signer(PayloadHashMode::IcebergRest); + let now = test_time(); + let hash_of = |builder: reqwest::RequestBuilder| { + let mut req = HttpRequest::new(builder.build().unwrap()); + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + req.headers() + .get("x-amz-content-sha256") + .unwrap() + .to_str() + .unwrap() + .to_string() + }; + + let client = reqwest::Client::new(); + let url = "https://rest.example.com/v1/namespaces"; + assert_eq!( + hash_of(client.post(url).body("")), + "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + ); + assert_eq!(hash_of(client.post(url)), EMPTY_HEX); + } + + #[test] + fn payload_hashes_standard_mode() { + let v = payload_hashes(Some(b"hello"), PayloadHashMode::StandardAws).0; + assert_eq!( + v, + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ); + } + + /// Pins a signature this crate produced; `signatures_match_iceberg_java` + /// checks against Java. + fn assert_signature_is(req: &HttpRequest, expected: &str) { + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!(auth.ends_with(&format!("Signature={expected}")), "{auth}"); + } + + fn test_signer(mode: PayloadHashMode) -> SigV4Signer { + SigV4Signer::builder() + .region("us-east-1") + .service("execute-api") + .mode(mode) + .build() + } + + fn test_credentials() -> Credentials { + Credentials::new("ak", "sk", None::, None, "test") + } + + fn example_credentials(session_token: Option<&str>) -> Credentials { + Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + session_token.map(str::to_string), + None, + "test", + ) + } + + fn test_time() -> DateTime { + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap() + } + + fn signed_headers(req: &HttpRequest) -> Vec { + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + let signed = auth.split("SignedHeaders=").nth(1).unwrap(); + let signed = signed.split(',').next().unwrap(); + signed.split(';').map(str::to_string).collect() + } + + /// Every header value as `(name, value)`, sorted. + fn header_list(req: &HttpRequest) -> Vec<(String, String)> { + let mut headers: Vec<_> = req + .headers() + .iter() + .map(|(n, v)| (n.to_string(), v.to_str().unwrap().to_string())) + .collect(); + headers.sort(); + headers + } + + fn sorted_pairs(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + let mut pairs: Vec<_> = pairs + .iter() + .map(|(n, v)| (n.to_string(), v.to_string())) + .collect(); + pairs.sort(); + pairs + } + + /// Records every event field. + #[derive(Clone, Default)] + struct CapturedLog(std::sync::Arc>); + + impl tracing::field::Visit for CapturedLog { + fn record_debug(&mut self, _: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0.lock().unwrap().push_str(&format!("{value:?}")); + } + } + + impl tracing::Subscriber for CapturedLog { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::Id { + tracing::Id::from_u64(1) + } + fn record(&self, _: &tracing::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::Id, _: &tracing::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + event.record(&mut self.clone()); + } + fn enter(&self, _: &tracing::Id) {} + fn exit(&self, _: &tracing::Id) {} + } + + /// `aws_sigv4` does not redact `Original-Authorization` itself. + #[test] + fn signing_does_not_trace_a_relocated_bearer_token() { + const TOKEN: &str = "Bearer topsecretdelegatetoken"; + let signer = test_signer(PayloadHashMode::IcebergRest); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header(reqwest::header::AUTHORIZATION, TOKEN) + .build() + .unwrap(), + ); + + let log = CapturedLog::default(); + tracing::subscriber::with_default(log.clone(), || { + // `sign_at` mutes only at this level, which is what is under test. + assert_eq!( + tracing::level_filters::LevelFilter::current(), + tracing::level_filters::LevelFilter::TRACE + ); + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + tracing::trace!(canary = "subscriber-is-live"); + }); + + let captured = log.0.lock().unwrap().clone(); + assert!(captured.contains("subscriber-is-live"), "captured nothing"); + assert!(!captured.contains(TOKEN), "{captured}"); + assert_eq!(req.headers().get(RELOCATED_AUTHORIZATION).unwrap(), TOKEN); + } + + #[test] + fn a_non_utf8_header_is_rejected_without_changing_the_request() { + let signer = test_signer(PayloadHashMode::IcebergRest); + for name in ["x-amz-meta-tenant", "authorization", "x-amz-content-sha256"] { + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://user:pw@rest.example.com/v1/config?warehouse=my+catalog" + .parse() + .unwrap(), + )); + req.headers_mut() + .insert("authorization", "Bearer delegate-token".parse().unwrap()); + req.headers_mut() + .insert("x-amz-content-sha256", "caller-hash".parse().unwrap()); + req.headers_mut().insert( + name, + reqwest::header::HeaderValue::from_bytes(b"acme\xfa").unwrap(), + ); + let headers = req.headers().clone(); + let url = req.url().clone(); + + let err = signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains(name), "{err}"); + assert_eq!(req.headers(), &headers, "{name}"); + assert_eq!(req.url(), &url, "{name}"); + } + } + + #[test] + fn a_non_ascii_utf8_header_is_signed() { + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + req.headers_mut().insert( + "x-tenant", + reqwest::header::HeaderValue::from_bytes("Zürich".as_bytes()).unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-tenant" + ]); + } + + #[test] + fn a_signing_failure_after_relocation_restores_the_request() { + let signer = test_signer(PayloadHashMode::IcebergRest); + // Longer than `http::Uri` accepts, so signing fails after the headers + // and query were rewritten. + let url = format!( + "https://rest.example.com/v1/config?warehouse=my+catalog&pad={}", + "a".repeat(70_000) + ); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get(url) + .header("authorization", "Bearer delegate-token") + .header("x-amz-content-sha256", "caller-hash") + .build() + .unwrap(), + ); + let headers = req.headers().clone(); + let url = req.url().clone(); + + let err = signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}"); + assert_eq!(req.headers(), &headers); + assert_eq!(req.url(), &url); + } + + /// Relocation appends to a caller's `Original-x-amz-*` after signing, so it + /// must stay unsigned. + #[test] + fn a_caller_supplied_relocation_header_is_not_signed() { + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("x-amz-content-sha256", "caller-hash") + .header("original-x-amz-content-sha256", "previous") + .build() + .unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date" + ]); + let relocated: Vec<_> = req + .headers() + .get_all("original-x-amz-content-sha256") + .iter() + .map(|v| v.to_str().unwrap()) + .collect(); + assert_eq!(relocated, ["previous", "caller-hash"]); + } + + #[test] + fn userinfo_is_stripped_before_signing() { + // A hand-built request can carry userinfo; the wire `Host` never does. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://user:pw@rest.example.com/v1/config" + .parse() + .unwrap(), + )); + assert_eq!(req.url().username(), "user"); + let now = test_time(); + + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + + assert_eq!(req.url().username(), ""); + assert_eq!(req.url().password(), None); + let mut plain = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + signer + .sign_at(&mut plain, &test_credentials(), now) + .unwrap(); + assert_eq!( + req.headers().get("authorization"), + plain.headers().get("authorization"), + ); + } + + #[test] + fn a_doubled_slash_in_the_path_is_normalized() { + // A trailing slash on the catalog URI gives `//v1/...`, which + // `Aws4Signer` collapses. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com//v1//config".parse().unwrap(), + )); + let now = test_time(); + + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + + let mut plain = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + signer + .sign_at(&mut plain, &test_credentials(), now) + .unwrap(); + assert_eq!( + req.headers().get("authorization"), + plain.headers().get("authorization"), + ); + } + + #[test] + fn caller_headers_the_signer_overwrites_are_relocated() { + // As in Java, conflicting caller values move to `Original-`. + let creds = Credentials::new( + "ak".to_string(), + "sk".to_string(), + Some("signer-token".to_string()), + None, + "test", + ); + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer caller-token") + .header("x-amz-date", "19700101T000000Z") + .header("x-amz-security-token", "caller-session") + .header("x-amz-content-sha256", "caller-hash") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert_eq!( + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=ak/20150830/us-east-1/execute-api/aws4_request, \ + SignedHeaders=host;original-authorization;x-amz-content-sha256;x-amz-date;\ + x-amz-security-token, \ + Signature=87560cb735277b284a547d63f57cfe3a721f9bce0b08a9402a58d754d2ed707c" + ), + ("original-authorization", "Bearer caller-token"), + ("original-x-amz-content-sha256", "caller-hash"), + ("original-x-amz-date", "19700101T000000Z"), + ("original-x-amz-security-token", "caller-session"), + ("x-amz-content-sha256", EMPTY_HEX), + ("x-amz-date", "20150830T123600Z"), + ("x-amz-security-token", "signer-token"), + ]) + ); + // Relocated originals may be credentials themselves. + for relocated in [RELOCATED_SECURITY_TOKEN, RELOCATED_CONTENT_SHA256] { + assert!(req.headers().get(relocated).unwrap().is_sensitive()); + } + } + + #[test] + fn an_existing_authorization_is_never_signed() { + // The signer replaces `authorization`, and a proxy may rewrite + // `user-agent`, so neither is signed. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer caller-token") + .header("user-agent", "example/1.0") + .build() + .unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "original-authorization", + "x-amz-content-sha256", + "x-amz-date" + ]); + } + + /// As in Java, every `Authorization` value is relocated, and stays redacted. + #[test] + fn every_repeated_authorization_is_relocated_and_kept_sensitive() { + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer first") + .header("authorization", "Bearer second") + .build() + .unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + let relocated: Vec<_> = req + .headers() + .get_all(RELOCATED_AUTHORIZATION) + .iter() + .collect(); + assert_eq!(relocated.len(), 2, "{relocated:?}"); + assert_eq!(relocated[0], "Bearer first"); + assert_eq!(relocated[1], "Bearer second"); + assert!(relocated.iter().all(|v| v.is_sensitive()), "{relocated:?}"); + } + + #[test] + fn hop_by_hop_headers_are_not_signed() { + // A proxy may drop or rewrite these, so they are not signed. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("expect", "100-continue") + .header("connection", "keep-alive") + .header("x-forwarded-for", "203.0.113.7") + .header("x-tenant", "acme") + .build() + .unwrap(), + ); + let now = test_time(); + + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-tenant" + ]); + assert_signature_is( + &req, + "f938221412ed6b55cf3db380ce6ded476419ad3d7db4c76d932031c98465ce79", + ); + } + + #[test] + fn signed_credentials_are_marked_sensitive() { + // Both carry a credential, so `Debug` must not print them. + let creds = Credentials::new( + "ak".to_string(), + "sk".to_string(), + Some("session-token".to_string()), + None, + "test", + ); + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert!(req.headers().get("authorization").unwrap().is_sensitive()); + assert!( + req.headers() + .get("x-amz-security-token") + .unwrap() + .is_sensitive() + ); + let debug = format!("{:?}", req.headers()); + assert!(!debug.contains("session-token"), "{debug}"); + } + + #[test] + fn signs_with_a_non_default_service_and_session_token() { + // As a non-AWS catalog might vend: its own signing name, and STS + // credentials. + let creds = Credentials::new( + "STS.EXAMPLEACCESSKEYID", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Some("example-session-token".to_string()), + None, + "test", + ); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("custom-service") + .mode(PayloadHashMode::IcebergRest) + .build(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://catalog.example.com/v1/config?warehouse=my-catalog") + .build() + .unwrap(), + ); + let now = Utc.with_ymd_and_hms(2026, 8, 26, 12, 0, 0).unwrap(); + + signer.sign_at(&mut req, &creds, now).unwrap(); + + assert_signature_is( + &req, + "6b7065e5f44da4f5c3654126b8d5fe29599905afb6b98f4de65b6ec6e1be783f", + ); + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!( + auth.contains("/us-east-1/custom-service/aws4_request"), + "{auth}" + ); + assert_eq!( + req.headers().get("x-amz-security-token").unwrap(), + "example-session-token" + ); + } + + #[test] + fn signs_request_iceberg_mode() { + let creds = example_credentials(Some("SESSIONTOKEN")); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); + let client = reqwest::Client::new(); + let mut req = HttpRequest::new( + client + .post("https://rest.example.com/v1/namespaces") + .body("{}") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert_eq!( + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/glue/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=effad6acde583dd14ba7aff52b2b83776a54421c010fe82067f166819057cb32" + ), + ( + "x-amz-content-sha256", + "RBNvo1WzZ4oRRq0W9+hknpT7T8If536DEMBg9hyq/4o=" + ), + ("x-amz-date", "20150830T123600Z"), + ("x-amz-security-token", "SESSIONTOKEN"), + ]) + ); + } + + /// An empty body hashes to the hex constant, and caller headers are signed + /// (Java's `authenticateWithoutBody`). + #[test] + fn signs_empty_body_and_all_headers() { + let creds = example_credentials(None); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); + let client = reqwest::Client::new(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com/v1/config") + .header("content-type", "application/json") + .header("content-encoding", "gzip") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert_eq!( + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/glue/aws4_request, \ + SignedHeaders=content-encoding;content-type;host;x-amz-content-sha256;x-amz-date, \ + Signature=eaef7eb88d9cd810031684748671d8a3c9394ea5168622a212ed041b670b9777" + ), + ("content-encoding", "gzip"), + ("content-type", "application/json"), + ("x-amz-content-sha256", EMPTY_HEX), + ("x-amz-date", "20150830T123600Z"), + ]) + ); + } + + #[test] + fn iceberg_mode_signs_the_hex_payload_hash_not_the_base64_header() { + // Needs a body: without one the header is the hex constant too. + let creds = example_credentials(None); + let signer = test_signer(PayloadHashMode::IcebergRest); + let body = br#"{"namespace":["ns"]}"#; + let mut req = HttpRequest::new( + reqwest::Client::new() + .post("https://rest.example.com/v1/namespaces") + .body(body.to_vec()) + .build() + .unwrap(), + ); + let now = test_time(); + + signer.sign_at(&mut req, &creds, now).unwrap(); + + assert_eq!( + req.headers().get("x-amz-content-sha256").unwrap(), + payload_hashes(Some(body), PayloadHashMode::IcebergRest) + .0 + .as_str() + ); + assert_ne!( + req.headers().get("x-amz-content-sha256").unwrap(), + encode_hex(&Sha256::digest(body)).as_str() + ); + assert_signature_is( + &req, + "c68682c26cab6a781256f83b0076f50014f4922c3907f4ff09c204a74d61fc1d", + ); + } + + /// What Iceberg Java's `RESTSigV4AuthSession` (iceberg-aws 1.10.1) sent for + /// these requests, given these credentials as `rest.*` properties. + #[test] + fn signatures_match_iceberg_java() { + let creds = example_credentials(Some("example-session-token")); + let signer = test_signer(PayloadHashMode::IcebergRest); + let now = Utc.with_ymd_and_hms(2026, 10, 2, 11, 40, 30).unwrap(); + let client = reqwest::Client::new(); + + // A body, so the header carries base64, and a token to relocate. + let mut post = HttpRequest::new( + client + .post("https://rest.example.com/v1/namespaces") + .header("content-type", "application/json") + .header("authorization", "Bearer delegate-token") + .body(r#"{"namespace":["a","b"],"properties":{}}"#) + .build() + .unwrap(), + ); + signer.sign_at(&mut post, &creds, now).unwrap(); + assert_eq!( + post.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261002/us-east-1/execute-api/aws4_request, \ + SignedHeaders=content-type;host;original-authorization;\ + x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=c5d6bfcecd19c5421e8696c67465f6909b5e09398608d8e79a9eb391e04de556" + ); + assert_eq!( + post.headers().get("x-amz-content-sha256").unwrap(), + "n1CB8Yl4L77sBwHTc5qTpUEhlsHHOXFzg0dV0fbOwXs=" + ); + + // No body, a multi-level namespace and an encoded query. + let mut get = HttpRequest::new( + client + .get("https://rest.example.com/v1/namespaces/a%1Fb/tables/x,y?pageToken=a%20b%2Fc") + .build() + .unwrap(), + ); + signer.sign_at(&mut get, &creds, now).unwrap(); + assert_eq!( + get.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261002/us-east-1/execute-api/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=5f6832ec82fc821b08cd3eac3a86ffc3e0c333571fd0a5796e43f2534f23329f" + ); + } + + /// The signed `host` keeps a non-default port, as on the wire. + #[test] + fn signs_host_with_non_default_port() { + let creds = example_credentials(None); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); + let client = reqwest::Client::new(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com:8181/v1/config") + .build() + .unwrap(), + ); + let now = test_time(); + + signer.sign_at(&mut req, &creds, now).unwrap(); + assert_signature_is( + &req, + "f7801a4ecac5fe6dcc4ee385223428ec4833f21d0c2fc10d2fb00694b2c0def7", + ); + } + + /// Like `Aws4Signer`, the canonical path is encoded again: `,` becomes `%2C` + /// and `%2C` becomes `%252C`. + #[test] + fn canonical_uri_is_aws_double_encoded() { + let creds = example_credentials(None); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); + let client = reqwest::Client::new(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com/v1/namespaces/a%2Cb/tables/x,y") + .build() + .unwrap(), + ); + let now = test_time(); + + signer.sign_at(&mut req, &creds, now).unwrap(); + assert_signature_is( + &req, + "4d966b6fc2dfb62be5a603e4e07e5dc85b2af1c7e6a181c5646bfbf38ddfa543", + ); + } + + /// The AWS SigV4 test suite's `post-x-www-form-urlencoded`, which signs a + /// hex `x-amz-content-sha256` like this mode. + #[test] + fn standard_mode_matches_the_aws_test_suite() { + let creds = example_credentials(None); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("service") + .mode(PayloadHashMode::StandardAws) + .build(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .post("https://example.amazonaws.com/") + .header("content-type", "application/x-www-form-urlencoded") + .header("content-length", "13") + .body("Param1=value1") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert_eq!( + req.headers().get("x-amz-content-sha256").unwrap(), + "9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e" + ); + assert_eq!( + req.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, \ + SignedHeaders=content-length;content-type;host;x-amz-content-sha256;x-amz-date, \ + Signature=d3875051da38690788ef43de4db0d8f280229d82040bfac253562e56c3f20e0b" + ); + } + + /// The one test on the live clock: everything else pins the time. + #[test] + fn sign_stamps_the_current_time() { + use chrono::SubsecRound; + + let signer = test_signer(PayloadHashMode::IcebergRest); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + let before = Utc::now(); + signer.sign(&mut req, &test_credentials()).unwrap(); + let after = Utc::now(); + + let date = req.headers().get("x-amz-date").unwrap().to_str().unwrap(); + let stamped = chrono::NaiveDateTime::parse_from_str(date, "%Y%m%dT%H%M%SZ") + .unwrap() + .and_utc(); + assert_eq!(date.len(), "YYYYMMDDTHHMMSSZ".len(), "{date}"); + // The header drops sub-seconds, so compare against a truncated start. + assert!( + before.trunc_subsecs(0) <= stamped && stamped <= after, + "{date}" + ); + } +} diff --git a/crates/catalog/rest/src/request.rs b/crates/catalog/rest/src/request.rs index 7a3c8e084c..8c6dd7f571 100644 --- a/crates/catalog/rest/src/request.rs +++ b/crates/catalog/rest/src/request.rs @@ -19,6 +19,8 @@ use http::{HeaderMap, Method}; use iceberg::Result; +#[cfg(feature = "sigv4")] +use reqwest::Url; use reqwest::{Request, RequestBuilder}; /// An outgoing REST request being authenticated by an @@ -48,6 +50,18 @@ impl HttpRequest { self.inner } + /// The request URL. + #[cfg(feature = "sigv4")] + pub(crate) fn url(&self) -> &Url { + self.inner.url() + } + + /// The mutable request URL, for signers that rewrite it. + #[cfg(feature = "sigv4")] + pub(crate) fn url_mut(&mut self) -> &mut Url { + self.inner.url_mut() + } + /// The request method. pub fn method(&self) -> &Method { self.inner.method() From b2e90754b1f9eaea741d353d51ae644aa9cef479 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:19:20 +0000 Subject: [PATCH 50/63] chore(deps): Bump fsspec from 2026.4.0 to 2026.6.0 in /bindings/python (#3366) Bumps [fsspec](https://github.com/fsspec/filesystem_spec) from 2026.4.0 to 2026.6.0. - [Commits](https://github.com/fsspec/filesystem_spec/compare/2026.4.0...2026.6.0) --- updated-dependencies: - dependency-name: fsspec dependency-version: 2026.6.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bindings/python/uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock index 1a532bbda7..5553a28728 100644 --- a/bindings/python/uv.lock +++ b/bindings/python/uv.lock @@ -225,11 +225,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2026.4.0" +version = "2026.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, ] [[package]] From e42840bc98cbf91a67c014162ed68036a8452d7a Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Sun, 11 Oct 2026 10:37:04 +0000 Subject: [PATCH 51/63] test(reader): widen FLBA decimal page-index coverage and clarify docs (#3386) test(reader): widen FLBA decimal page-index coverage and clarify guard docs Address review feedback on #3328: - Reframe the width-guard doc and inline comment as defence-in-depth against a non-conforming writer - Hoist the duplicated min/max decode into a single closure. - Add page-index tests --- .../src/expr/visitors/page_index_evaluator.rs | 178 +++++++++++++++--- 1 file changed, 152 insertions(+), 26 deletions(-) diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index 88d1eea9ff..4e51042336 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -438,28 +438,20 @@ impl<'a> PageIndexEvaluator<'a> { .enumerate() .zip(row_counts.iter()) { - // A bound that won't decode (e.g. a min/max stat truncated - // by column_index_truncate_length, whose byte prefix no - // longer preserves the two's-complement decimal ordering) + // An undecodable bound (only a non-conforming writer + // produces one, see fixed_len_byte_array_decimal_bound_to_datum) // means this column's page index can't be trusted, so skip // pruning for the whole column rather than abort the scan. + let decode = |val: &[u8]| { + Self::fixed_len_byte_array_decimal_bound_to_datum( + field_type, + val, + type_length, + ) + }; let (min, max) = match ( - min.map(|val| { - Self::fixed_len_byte_array_decimal_bound_to_datum( - field_type, - val, - type_length, - ) - }) - .transpose(), - max.map(|val| { - Self::fixed_len_byte_array_decimal_bound_to_datum( - field_type, - val, - type_length, - ) - }) - .transpose(), + min.map(decode).transpose(), + max.map(decode).transpose(), ) { (Ok(min), Ok(max)) => (min, max), (Err(err), _) | (_, Err(err)) => { @@ -548,9 +540,11 @@ impl<'a> PageIndexEvaluator<'a> { /// Parquet stores Iceberg decimals with precision > 18 as big-endian /// two's-complement `FIXED_LEN_BYTE_ARRAY` of width `type_length`. /// - /// Returns an error for a bound whose width differs from `type_length`: that - /// signals a `column_index_truncate_length` truncation, and a truncated byte - /// prefix decodes to the wrong number under two's-complement. + /// Returns an error for a bound whose width differs from `type_length`. + /// Conforming writers never truncate a decimal column index (a truncated + /// two's-complement prefix would not preserve numeric order), so a short + /// bound comes from a non-conforming writer and would decode to the wrong + /// value. Reject it rather than prune on a bad bound. fn fixed_len_byte_array_decimal_bound_to_datum( field_type: &PrimitiveType, bytes: &[u8], @@ -1241,6 +1235,18 @@ mod tests { precision: u8, scale: i8, unscaled: &[i128], + ) -> Result<(Arc, NamedTempFile)> { + let pages: Vec> = unscaled.iter().copied().map(Some).collect(); + create_decimal_parquet_file_with_null_pages(precision, scale, &pages) + } + + /// Like [`create_decimal_parquet_file`], but each entry in `pages` is one + /// 1024-row page: `Some(v)` writes that repeated unscaled value, `None` + /// writes an all-null page (whose column index carries no min/max bound). + fn create_decimal_parquet_file_with_null_pages( + precision: u8, + scale: i8, + pages: &[Option], ) -> Result<(Arc, NamedTempFile)> { let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( "col_decimal", @@ -1258,11 +1264,16 @@ mod tests { let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); - for &unscaled in unscaled { + for page in pages { let array = Arc::new( - Decimal128Array::from_iter_values(std::iter::repeat_n(unscaled, 1024)) - .with_precision_and_scale(precision, scale) - .unwrap(), + match page { + Some(unscaled) => { + Decimal128Array::from_iter_values(std::iter::repeat_n(*unscaled, 1024)) + } + None => Decimal128Array::from(vec![None; 1024]), + } + .with_precision_and_scale(precision, scale) + .unwrap(), ) as ArrayRef; let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); // Write rows one at a time so the writer splits into per-value pages. @@ -2186,6 +2197,121 @@ mod tests { Ok(()) } + #[test] + fn eval_inequality_prunes_precision_38_fixed_len_byte_array_decimal_pages() -> Result<()> { + // precision 38 is the max Iceberg decimal precision -> 16-byte + // FIXED_LEN_BYTE_ARRAY bounds, the width extreme where decoding fills + // the full i128 with no sign-extension padding. Values near +/-(10^38-1) + // carry significant bits in the high-order bytes, and span both signs. + let max = 10_i128.pow(38) - 1; + let (metadata, _temp_file) = create_decimal_parquet_file(38, 2, &[-max, 0, max])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(38, 2)?; + + // `> 0` keeps only the +max page; the -max and 0 pages are pruned. + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(0, 2, 38)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(1024) + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_precision_widened_fixed_len_byte_array_decimal_pages() -> Result<()> { + // Precision widening keeps a file's original FIXED_LEN_BYTE_ARRAY width, + // so a decimal(20,2) file's 9-byte bounds are read here as decimal(30,2). + // The guard keys off the file's type_length (9), not the field + // precision's canonical width (13). + let (metadata, _temp_file) = create_decimal_parquet_file(20, 2, &[-400, 100, 300])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + // Read the 9-byte column as decimal(30,2). + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + // Pages hold -4.00, 1.00, 3.00. `> 1.50` keeps only the 3.00 page. + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(150, 2, 30)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(1024) + ]); + + Ok(()) + } + + #[test] + fn eval_is_null_selects_all_null_fixed_len_byte_array_decimal_page() -> Result<()> { + // precision 30 -> FIXED_LEN_BYTE_ARRAY page bounds. Page 0 is all-null, + // so its column index carries no min/max; the arm must pass those absent + // bounds straight to the predicate rather than try to decode them. Pages + // 1-2 hold 2.00 and 3.00. + let (metadata, _temp_file) = + create_decimal_parquet_file_with_null_pages(30, 2, &[None, Some(200), Some(300)])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + + // The column is optional so IS NULL binds to a real predicate (on a + // required field it would simplify to always-false). + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_decimal", + Type::Primitive(PrimitiveType::Decimal { + precision: 30, + scale: 2, + }), + false, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + // IS NULL keeps the all-null page and prunes the two value pages. + let filter = Reference::new("col_decimal") + .is_null() + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::select(1024), + RowSelector::skip(2048) + ]); + + Ok(()) + } + /// Builds a decimal query [`Datum`] from an `unscaled` mantissa, matching /// the field's `scale` and `precision`. fn decimal_datum(unscaled: i128, scale: u32, precision: u32) -> Result { From d63e8610f2152158882f8773f3745f3f32752912 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sun, 11 Oct 2026 13:53:01 +0000 Subject: [PATCH 52/63] fix(scan): reject ambiguous case-insensitive fields (#3070) * fix(scan): reject ambiguous case-insensitive fields Co-authored-by: Codex * fix(schema): clarify ambiguous case-insensitive lookup Expose the checked lookup, document collision semantics, and cover the public fallback. Simplify the binding and scan callers. Co-authored-by: Codex * fix(schema): improve case-insensitive collision diagnostics List sorted colliding field names with invalid_data!, simplify optional lookup, and loosen error assertions while checking the reported colliders. Co-authored-by: Codex --------- Co-authored-by: Codex --- crates/iceberg/public-api.txt | 1 + crates/iceberg/src/expr/term.rs | 19 +++++- crates/iceberg/src/scan/mod.rs | 68 +++++++++++++++++++-- crates/iceberg/src/spec/schema/mod.rs | 86 ++++++++++++++++++++++++--- 4 files changed, 160 insertions(+), 14 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 8a34bbf7b5..4666147573 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2474,6 +2474,7 @@ pub fn iceberg::spec::Schema::field_by_alias(&self, alias: &str) -> core::option pub fn iceberg::spec::Schema::field_by_id(&self, field_id: i32) -> core::option::Option<&iceberg::spec::NestedFieldRef> pub fn iceberg::spec::Schema::field_by_name(&self, field_name: &str) -> core::option::Option<&iceberg::spec::NestedFieldRef> pub fn iceberg::spec::Schema::field_by_name_case_insensitive(&self, field_name: &str) -> core::option::Option<&iceberg::spec::NestedFieldRef> +pub fn iceberg::spec::Schema::field_by_name_case_insensitive_checked(&self, field_name: &str) -> iceberg::Result> pub fn iceberg::spec::Schema::field_id_by_name(&self, name: &str) -> core::option::Option pub fn iceberg::spec::Schema::field_id_to_fields(&self) -> &std::collections::hash::map::HashMap pub fn iceberg::spec::Schema::field_id_to_name_map(&self) -> &std::collections::hash::map::HashMap diff --git a/crates/iceberg/src/expr/term.rs b/crates/iceberg/src/expr/term.rs index 4a5aedf743..7c5cc08919 100644 --- a/crates/iceberg/src/expr/term.rs +++ b/crates/iceberg/src/expr/term.rs @@ -313,7 +313,7 @@ impl Bind for Reference { let field = if case_sensitive { schema.field_by_name(&self.name) } else { - schema.field_by_name_case_insensitive(&self.name) + schema.field_by_name_case_insensitive_checked(&self.name)? }; let field = @@ -428,6 +428,23 @@ mod tests { assert_eq!(expected_ref, reference); } + #[test] + fn test_bind_reference_case_insensitive_ambiguous() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", PrimitiveType::Int.into()).into(), + NestedField::optional(2, "ID", PrimitiveType::Int.into()).into(), + ]) + .build() + .unwrap(), + ); + + let error = Reference::new("Id").bind(schema, false).unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::DataInvalid); + assert!(error.message().contains("case-insensitively"), "{error}"); + } + #[test] fn test_bind_reference_failure() { let schema = table_schema_simple(); diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 0c241125dc..7e81900a5b 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -53,13 +53,16 @@ use crate::{Error, ErrorKind, Result}; pub type ArrowRecordBatchStream = BoxStream<'static, Result>; /// Resolves a column name to its field ID, honouring the scan's case sensitivity. -fn resolve_field_id(schema: &Schema, column_name: &str, case_sensitive: bool) -> Option { +fn resolve_field_id( + schema: &Schema, + column_name: &str, + case_sensitive: bool, +) -> Result> { if case_sensitive { - schema.field_id_by_name(column_name) + Ok(schema.field_id_by_name(column_name)) } else { - schema - .field_by_name_case_insensitive(column_name) - .map(|field| field.id) + let field = schema.field_by_name_case_insensitive_checked(column_name)?; + Ok(field.map(|field| field.id)) } } @@ -79,7 +82,7 @@ fn collect_scan_field_ids( return get_metadata_field_id(column_name); } - let field_id = resolve_field_id(schema, column_name, case_sensitive).ok_or_else(|| { + let field_id = resolve_field_id(schema, column_name, case_sensitive)?.ok_or_else(|| { invalid_data!("Column {column_name} not found in table. Schema: {schema}") })?; @@ -692,6 +695,7 @@ mod tests { use futures::{TryStreamExt, stream}; use uuid::Uuid; + use super::resolve_field_id; use crate::arrow::ArrowReaderBuilder; use crate::expr::{BoundPredicate, Reference}; use crate::io::FileIO; @@ -761,6 +765,58 @@ mod tests { ); } + #[test] + fn test_resolve_field_id_case_insensitive_rejects_ambiguous_name() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", PrimitiveType::Int.into()).into(), + NestedField::optional(2, "ID", PrimitiveType::Int.into()).into(), + NestedField::optional(3, "data", PrimitiveType::Int.into()).into(), + ]) + .build() + .unwrap(); + + assert_eq!(resolve_field_id(&schema, "id", true).unwrap(), Some(1)); + assert_eq!(resolve_field_id(&schema, "DATA", false).unwrap(), Some(3)); + let error = resolve_field_id(&schema, "Id", false).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.message().contains("case-insensitively"), "{error}"); + } + + #[test] + fn test_case_insensitive_scan_with_ambiguous_schema() { + let fixture = TableTestFixture::new(); + let mut metadata = fixture.table.metadata().clone(); + let schema = Schema::builder() + .with_schema_id(metadata.current_schema_id()) + .with_fields(vec![ + NestedField::optional(1, "id", PrimitiveType::Int.into()).into(), + NestedField::optional(2, "ID", PrimitiveType::Int.into()).into(), + NestedField::optional(3, "data", PrimitiveType::Int.into()).into(), + ]) + .build() + .unwrap(); + metadata + .schemas + .insert(metadata.current_schema_id(), Arc::new(schema)); + let table = fixture.table.with_metadata(Arc::new(metadata)); + + table + .scan() + .with_case_sensitive(false) + .select(["DATA"]) + .build() + .unwrap(); + let error = table + .scan() + .with_case_sensitive(false) + .select(["Id"]) + .build() + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.message().contains("case-insensitively"), "{error}"); + } + #[tokio::test] async fn test_case_insensitive_scan_resolves_mismatched_column_case() { let mut fixture = TableTestFixture::new(); diff --git a/crates/iceberg/src/spec/schema/mod.rs b/crates/iceberg/src/spec/schema/mod.rs index c6dcf07079..67924dfe8f 100644 --- a/crates/iceberg/src/spec/schema/mod.rs +++ b/crates/iceberg/src/spec/schema/mod.rs @@ -71,7 +71,8 @@ pub struct Schema { id_to_field: HashMap, name_to_id: HashMap, - lowercase_name_to_id: HashMap, + // `None` marks a lower-cased name that resolves to more than one field. + lowercase_name_to_id: HashMap>, id_to_name: HashMap, field_id_to_accessor: HashMap>, @@ -150,10 +151,13 @@ impl SchemaBuilder { index.indexes() }; - let lowercase_name_to_id = name_to_id - .iter() - .map(|(k, v)| (k.to_lowercase(), *v)) - .collect(); + let mut lowercase_name_to_id = HashMap::new(); + for (name, field_id) in &name_to_id { + lowercase_name_to_id + .entry(name.to_lowercase()) + .and_modify(|existing| *existing = None) + .or_insert(Some(*field_id)); + } let highest_field_id = id_to_field.keys().max().cloned().unwrap_or(0); @@ -342,13 +346,47 @@ impl Schema { .and_then(|id| self.field_by_id(*id)) } - /// Get field by field name, but in case-insensitive way. + /// Get field by field name in a case-insensitive way. /// /// Both full name and short name could work here. + /// Returns `None` if the name is missing or matches multiple fields case-insensitively. + /// Use [`Self::field_by_name_case_insensitive_checked`] to distinguish those cases. pub fn field_by_name_case_insensitive(&self, field_name: &str) -> Option<&NestedFieldRef> { self.lowercase_name_to_id .get(&field_name.to_lowercase()) - .and_then(|id| self.field_by_id(*id)) + .copied() + .flatten() + .and_then(|id| self.field_by_id(id)) + } + + /// Get field by field name in a case-insensitive way, reporting ambiguous names. + /// + /// Both full name and short name could work here. Returns `Ok(None)` if the name is + /// missing, or an error if it matches multiple fields case-insensitively. A collision + /// on another name does not prevent this name from resolving. This differs from Iceberg + /// Java, which rejects all case-insensitive lookups after a collision, and PyIceberg, + /// which silently keeps one of the colliding fields. + pub fn field_by_name_case_insensitive_checked( + &self, + field_name: &str, + ) -> Result> { + let lowercase_name = field_name.to_lowercase(); + match self.lowercase_name_to_id.get(&lowercase_name) { + Some(Some(id)) => Ok(self.field_by_id(*id)), + Some(None) => { + let mut colliders: Vec<&str> = self + .name_to_id + .keys() + .filter(|name| name.to_lowercase() == lowercase_name) + .map(String::as_str) + .collect(); + colliders.sort_unstable(); + Err(invalid_data!( + "Multiple fields match {field_name} case-insensitively: {colliders:?}" + )) + } + None => Ok(None), + } } /// Get field by alias. @@ -973,6 +1011,40 @@ table { } } + #[test] + fn test_schema_field_by_name_case_insensitive_checked() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "ID", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "data", Primitive(PrimitiveType::String)).into(), + ]) + .build() + .expect("case-insensitive name ambiguity is valid in a schema"); + + assert_eq!( + schema + .field_by_name_case_insensitive_checked("DATA") + .unwrap() + .map(|field| field.id), + Some(3) + ); + assert!( + schema + .field_by_name_case_insensitive_checked("missing") + .unwrap() + .is_none() + ); + + let error = schema + .field_by_name_case_insensitive_checked("Id") + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::DataInvalid); + assert!(error.message().contains("case-insensitively"), "{error}"); + assert!(error.message().contains("[\"ID\", \"id\"]"), "{error}"); + assert!(schema.field_by_name_case_insensitive("Id").is_none()); + } + #[test] fn test_schema_find_column_name() { let expected_column_name = HashMap::from([ From a76678363835a3035c38b9ab53785e9d9b762d24 Mon Sep 17 00:00:00 2001 From: Xander Date: Sun, 11 Oct 2026 14:25:41 +0000 Subject: [PATCH 53/63] feat(RTAS) Assign fresh ids (#3066) * feat(RTAS) Assign fresh ids * update tests with more types * avoid clone * document assign better * relax test * additional test * comment * Add debug assert * lowercase * doc * not nested * test * test comments * comments * starts_from --- .../src/spec/schema/assign_fresh_ids.rs | 520 ++++++++++++++++++ .../iceberg/src/spec/schema/id_reassigner.rs | 4 +- crates/iceberg/src/spec/schema/mod.rs | 3 + 3 files changed, 525 insertions(+), 2 deletions(-) create mode 100644 crates/iceberg/src/spec/schema/assign_fresh_ids.rs diff --git a/crates/iceberg/src/spec/schema/assign_fresh_ids.rs b/crates/iceberg/src/spec/schema/assign_fresh_ids.rs new file mode 100644 index 0000000000..94f3030681 --- /dev/null +++ b/crates/iceberg/src/spec/schema/assign_fresh_ids.rs @@ -0,0 +1,520 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::utils::try_insert_field; +use super::*; +use crate::error::invalid_data; + +/// Reassigns `schema`'s field ids, reusing ids from `base` for fields whose full name is unchanged +/// and drawing fresh ids from `last_column_id + 1` upwards for everything else. +/// +/// `last_column_id` must be every id the table has ever assigned, not merely `base`'s highest: pass +/// `table_metadata.last_column_id()`, which also reserves the ids of columns already dropped from +/// `base`. A reused id does not consume a fresh one, so a too-low `last_column_id` fails in one of +/// two ways. Falling below `base`'s own ids is rejected here. Landing on the id of a column already +/// dropped from `base` cannot be detected from `base` alone: nothing here or in `build()` notices, +/// and a new column silently inherits a retired id. Passing a schema's `highest_field_id()` instead +/// of the table's `last_column_id()` is the usual way to hit the latter. +/// +/// The returned `schema_id` is carried over unchanged and is not authoritative; it is arbitrated by +/// [`TableMetadataBuilder::add_schema`](crate::spec::TableMetadataBuilder::add_schema). +pub(crate) fn assign_fresh_ids( + schema: Schema, + base: &Schema, + last_column_id: i32, +) -> Result { + if last_column_id < base.highest_field_id() { + return Err(invalid_data!( + "last_column_id ({last_column_id}) is below base.highest_field_id() ({}); pass table_metadata.last_column_id()", + base.highest_field_id() + )); + } + + let start_from = last_column_id + .checked_add(1) + .ok_or_else(|| invalid_data!("Field ID overflowed, cannot add more fields"))?; + + let Schema { + r#struct, + schema_id, + identifier_field_ids, + alias_to_id, + id_to_name, + .. + } = schema; + let mut assigner = AssignFreshIds::new(id_to_name, base, start_from); + let fields = assigner.assign_fields(r#struct.fields().to_vec())?; + let identifier_field_ids = assigner.apply_to_identifier_fields(identifier_field_ids)?; + let alias_to_id = assigner.apply_to_aliases(alias_to_id)?; + + Schema::builder() + .with_schema_id(schema_id) + .with_fields(fields) + .with_identifier_field_ids(identifier_field_ids) + .with_alias(alias_to_id) + .build() +} + +struct AssignFreshIds { + next_field_id: i32, + target_names: HashMap, + base_ids: HashMap, + old_to_new_id: HashMap, +} + +impl AssignFreshIds { + fn new(target_names: HashMap, base: &Schema, start_from: i32) -> Self { + Self { + next_field_id: start_from, + target_names, + base_ids: base + .field_id_to_name_map() + .iter() + .map(|(id, name)| (name.clone(), *id)) + .collect(), + old_to_new_id: HashMap::new(), + } + } + + /// Returns `base`'s id when the field's full name is unchanged, otherwise consumes a fresh id by + /// advancing `next_field_id`. + fn resolve_or_assign_id(&mut self, old_id: i32) -> Result { + if let Some(id) = self + .target_names + .get(&old_id) + .and_then(|name| self.base_ids.get(name)) + { + return Ok(*id); + } + + let id = self.next_field_id; + self.next_field_id = self + .next_field_id + .checked_add(1) + .ok_or_else(|| invalid_data!("Field ID overflowed, cannot add more fields"))?; + Ok(id) + } + + fn assign_fields(&mut self, fields: Vec) -> Result> { + let outer_fields = fields + .into_iter() + .map(|field| { + let new_id = self.resolve_or_assign_id(field.id)?; + try_insert_field(&mut self.old_to_new_id, field.id, new_id)?; + Ok(Arc::new(Arc::unwrap_or_clone(field).with_id(new_id))) + }) + .collect::>>()?; + + outer_fields + .into_iter() + .map(|field| { + if !field.field_type.is_nested() { + Ok(field) + } else { + let mut field = Arc::unwrap_or_clone(field); + *field.field_type = self.assign_type(*field.field_type)?; + Ok(Arc::new(field)) + } + }) + .collect() + } + + fn assign_type(&mut self, field_type: Type) -> Result { + match field_type { + Type::Primitive(primitive) => Ok(Type::Primitive(primitive)), + Type::Struct(r#struct) => Ok(Type::Struct(StructType::new( + self.assign_fields(r#struct.fields().to_vec())?, + ))), + Type::List(list) => { + let new_id = self.resolve_or_assign_id(list.element_field.id)?; + try_insert_field(&mut self.old_to_new_id, list.element_field.id, new_id)?; + let mut element_field = Arc::unwrap_or_clone(list.element_field); + element_field.id = new_id; + *element_field.field_type = self.assign_type(*element_field.field_type)?; + Ok(Type::List(ListType { + element_field: Arc::new(element_field), + })) + } + Type::Map(map) => { + // Key and value ids are resolved before recursing into either, matching Java and + // intentionally unlike `ReassignFieldIds`, which recurses the key first. + let new_key_id = self.resolve_or_assign_id(map.key_field.id)?; + let new_value_id = self.resolve_or_assign_id(map.value_field.id)?; + try_insert_field(&mut self.old_to_new_id, map.key_field.id, new_key_id)?; + try_insert_field(&mut self.old_to_new_id, map.value_field.id, new_value_id)?; + + let mut key_field = Arc::unwrap_or_clone(map.key_field); + key_field.id = new_key_id; + *key_field.field_type = self.assign_type(*key_field.field_type)?; + + let mut value_field = Arc::unwrap_or_clone(map.value_field); + value_field.id = new_value_id; + *value_field.field_type = self.assign_type(*value_field.field_type)?; + + Ok(Type::Map(MapType { + key_field: Arc::new(key_field), + value_field: Arc::new(value_field), + })) + } + Type::Variant(variant) => Ok(Type::Variant(variant)), + } + } + + fn apply_to_identifier_fields(&self, field_ids: HashSet) -> Result> { + field_ids + .into_iter() + .map(|id| { + self.old_to_new_id + .get(&id) + .copied() + .ok_or_else(|| invalid_data!("identifier field id {id} not found")) + }) + .collect() + } + + fn apply_to_aliases(&self, aliases: BiHashMap) -> Result> { + aliases + .into_iter() + .map(|(name, id)| { + self.old_to_new_id + .get(&id) + .copied() + .ok_or_else(|| invalid_data!("Field with id {id} for alias {name} not found")) + .map(|new_id| (name, new_id)) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::{Literal, VariantType}; + + fn empty_schema() -> Schema { + Schema::builder().build().unwrap() + } + + #[test] + fn test_assign_fresh_ids() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(0, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(1, "c", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(23)) + .with_write_default(Literal::int(34)) + .into(), + NestedField::required(2, "B", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let expected = Schema::builder() + .with_fields(vec![ + NestedField::required(11, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(12, "c", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(23)) + .with_write_default(Literal::int(34)) + .into(), + NestedField::required(13, "B", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let assigned = assign_fresh_ids(schema, &empty_schema(), 10).unwrap(); + + assert_eq!(assigned.as_struct(), expected.as_struct()); + } + + #[test] + fn test_assign_fresh_ids_with_type() { + for test_type in [ + Type::Primitive(PrimitiveType::Boolean), + Type::Primitive(PrimitiveType::Int), + Type::Primitive(PrimitiveType::Long), + Type::Primitive(PrimitiveType::Float), + Type::Primitive(PrimitiveType::Double), + Type::Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }), + Type::Primitive(PrimitiveType::Date), + Type::Primitive(PrimitiveType::Time), + Type::Primitive(PrimitiveType::Timestamp), + Type::Primitive(PrimitiveType::Timestamptz), + Type::Primitive(PrimitiveType::TimestampNs), + Type::Primitive(PrimitiveType::TimestamptzNs), + Type::Primitive(PrimitiveType::String), + Type::Primitive(PrimitiveType::Uuid), + Type::Primitive(PrimitiveType::Fixed(16)), + Type::Primitive(PrimitiveType::Binary), + Type::Variant(VariantType), + ] { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(0, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(1, "data", test_type.clone()).into(), + ]) + .build() + .unwrap(); + let expected = Schema::builder() + .with_fields(vec![ + NestedField::required(11, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(12, "data", test_type.clone()).into(), + ]) + .build() + .unwrap(); + + let assigned = assign_fresh_ids(schema, &empty_schema(), 10).unwrap(); + + assert_eq!( + assigned.as_struct(), + expected.as_struct(), + "failed for type: {test_type:?}" + ); + } + } + + #[test] + fn test_assign_fresh_ids_reuses_full_names_and_assigns_new_ids_in_correct_order() { + let base = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "nested", + Type::Struct(StructType::new(vec![ + NestedField::optional(2, "a", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ) + .into(), + NestedField::optional( + 3, + "items", + Type::List(ListType::new( + NestedField::list_element(4, Type::Primitive(PrimitiveType::String), false) + .into(), + )), + ) + .into(), + NestedField::optional( + 5, + "properties", + Type::Map(MapType::optional( + 6, + Type::Primitive(PrimitiveType::String), + 7, + Type::Primitive(PrimitiveType::Long), + )), + ) + .into(), + NestedField::required(8, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(9, "dropped", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let replacement = Schema::builder() + .with_schema_id(1) + .with_identifier_field_ids([18]) + .with_alias(BiHashMap::from_iter([("a_alias".to_string(), 12)])) + .with_fields(vec![ + NestedField::required( + 10, + "nested", + Type::Struct(StructType::new(vec![ + NestedField::optional(11, "b", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(12, "a", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ) + .into(), + NestedField::optional( + 13, + "items", + Type::List(ListType::new( + NestedField::list_element( + 14, + Type::Primitive(PrimitiveType::String), + false, + ) + .into(), + )), + ) + .into(), + NestedField::optional( + 15, + "properties", + Type::Map(MapType::optional( + 16, + Type::Primitive(PrimitiveType::String), + 17, + Type::Primitive(PrimitiveType::Long), + )), + ) + .into(), + NestedField::required(18, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(19, "z", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let assigned = assign_fresh_ids(replacement, &base, 9).unwrap(); + + assert_eq!(assigned.field_by_name("nested").unwrap().id, 1); + assert_eq!(assigned.field_by_name("nested.a").unwrap().id, 2); + assert_eq!(assigned.field_by_name("items").unwrap().id, 3); + assert_eq!(assigned.field_by_name("items.element").unwrap().id, 4); + assert_eq!(assigned.field_by_name("properties").unwrap().id, 5); + assert_eq!(assigned.field_by_name("properties.key").unwrap().id, 6); + assert_eq!(assigned.field_by_name("properties.value").unwrap().id, 7); + assert_eq!(assigned.field_by_name("x").unwrap().id, 8); + assert_eq!(assigned.field_by_name("z").unwrap().id, 10); + assert_eq!(assigned.field_by_name("nested.b").unwrap().id, 11); + assert_eq!( + assigned.identifier_field_ids().collect::>(), + HashSet::from([8]) + ); + assert_eq!(assigned.field_by_alias("a_alias").unwrap().id, 2); + assert_eq!(assigned.highest_field_id(), 11); + } + + /// The table once had ids 1..=5; only 1 and 3 survive in `base`, so 4 and 5 are retired and the + /// correct `last_column_id` is 5, not `base.highest_field_id()`. + fn schemas_with_dropped_column_ids() -> (Schema, Schema) { + let base = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(3, "b", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let replacement = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "fresh", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + (base, replacement) + } + + #[test] + fn test_assign_fresh_ids_skips_dropped_column_ids_when_seeded_correctly() { + let (base, replacement) = schemas_with_dropped_column_ids(); + + let assigned = assign_fresh_ids(replacement, &base, 5).unwrap(); + + assert_eq!(assigned.field_by_name("a").unwrap().id, 1); + assert_eq!(assigned.field_by_name("fresh").unwrap().id, 6); + } + + #[test] + fn test_assign_fresh_ids_seeded_too_low_reuses_dropped_column_id() { + let (base, replacement) = schemas_with_dropped_column_ids(); + + // Passing base's highest id as `last_column_id` clears the guard and `build()` sees no + // duplicates, so the retired id 4 is reused silently. `base` alone cannot reveal that 4 is + // retired, so this stays the caller's contract. + let assigned = assign_fresh_ids(replacement, &base, base.highest_field_id()).unwrap(); + + assert_eq!(assigned.field_by_name("fresh").unwrap().id, 4); + } + + #[test] + fn test_assign_fresh_ids_rejects_last_column_id_below_base_highest_field_id() { + let (base, replacement) = schemas_with_dropped_column_ids(); + + let err = assign_fresh_ids(replacement, &base, base.highest_field_id() - 1).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("last_column_id (2) is below base.highest_field_id() (3)") + ); + } + + #[test] + fn test_assign_fresh_ids_rejects_alias_for_dropped_field() { + // `build()` validates `identifier_field_ids` but not `alias_to_id`, so an alias left + // pointing at a field the replacement drops reaches `apply_to_aliases`. + let base = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "dropped", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let replacement = Schema::builder() + .with_alias(BiHashMap::from_iter([("dropped_alias".to_string(), 2)])) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let err = assign_fresh_ids(replacement, &base, 2).unwrap_err(); + + assert!( + err.message() + .contains("Field with id 2 for alias dropped_alias not found") + ); + } + + #[test] + fn test_assign_fresh_ids_assigns_map_ids_before_nested_types() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "map", + Type::Map(MapType::required( + 2, + Type::Struct(StructType::new(vec![ + NestedField::required( + 3, + "key_nested", + Type::Primitive(PrimitiveType::Int), + ) + .into(), + ])), + 4, + Type::Struct(StructType::new(vec![ + NestedField::required( + 5, + "value_nested", + Type::Primitive(PrimitiveType::Int), + ) + .into(), + ])), + )), + ) + .into(), + NestedField::required(6, "tail", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let assigned = assign_fresh_ids(schema, &empty_schema(), 10).unwrap(); + + assert_eq!(assigned.field_by_name("map").unwrap().id, 11); + assert_eq!(assigned.field_by_name("tail").unwrap().id, 12); + assert_eq!(assigned.field_by_name("map.key").unwrap().id, 13); + assert_eq!(assigned.field_by_name("map.value").unwrap().id, 14); + assert_eq!(assigned.field_by_name("map.key.key_nested").unwrap().id, 15); + assert_eq!( + assigned.field_by_name("map.value.value_nested").unwrap().id, + 16 + ); + } +} diff --git a/crates/iceberg/src/spec/schema/id_reassigner.rs b/crates/iceberg/src/spec/schema/id_reassigner.rs index 0c9b4de3f9..dafaeaa641 100644 --- a/crates/iceberg/src/spec/schema/id_reassigner.rs +++ b/crates/iceberg/src/spec/schema/id_reassigner.rs @@ -53,7 +53,7 @@ impl ReassignFieldIds { outer_fields .into_iter() .map(|field| { - if field.field_type.is_primitive() { + if !field.field_type.is_nested() { Ok(field) } else { let mut new_field = Arc::unwrap_or_clone(field); @@ -122,7 +122,7 @@ impl ReassignFieldIds { self.old_to_new_id .get(&id) .copied() - .ok_or_else(|| invalid_data!("Identifier Field ID {id} not found")) + .ok_or_else(|| invalid_data!("identifier field id {id} not found")) }) .collect() } diff --git a/crates/iceberg/src/spec/schema/mod.rs b/crates/iceberg/src/spec/schema/mod.rs index 67924dfe8f..d55d0889d7 100644 --- a/crates/iceberg/src/spec/schema/mod.rs +++ b/crates/iceberg/src/spec/schema/mod.rs @@ -25,6 +25,9 @@ mod utils; mod visitor; pub use self::visitor::*; pub(super) mod _serde; +// TODO: Use for table replacement in https://github.com/apache/iceberg-rust/issues/3056. +#[allow(dead_code)] +pub(crate) mod assign_fresh_ids; mod id_reassigner; mod index; mod prune_columns; From 050ddfda869068cf2d19683e45d447314181415b Mon Sep 17 00:00:00 2001 From: Ethan Woodbury Date: Sun, 11 Oct 2026 18:16:21 +0000 Subject: [PATCH 54/63] fix(arrow): reconcile evolved nested structs by field id, not position (#3255) * fix(arrow): reconcile evolved nested structs by field id, not position When a struct gains a field (schema evolution) after a data file was written, the file's struct has fewer children than the table schema. RecordBatchTransformer promoted such columns with arrow_cast::cast, which matches struct/list/map children positionally -- so every child after the gap shifts and types collide, e.g. 'Casting from Utf8 to Struct' when a scalar lands on an added list. Replace the flat cast with a PromotePlan that is resolved once per file when the BatchTransform is built: nested struct children are matched to the target schema by PARQUET:field_id, recursing through list/large-list/map, so the per-batch work is just index lookups and array assembly. Absent optional children are filled with typed nulls; absent required children error, as do absent children with an initial-default (not yet wired up for nested fields) and source structs whose children carry no field ids, rather than silently producing wrong data. Primitives still use cast for valid Iceberg promotions. Mirrors iceberg-java's by-field-id nested readers. Closes #2617 * test(arrow): cover appended nested struct field on promote Add a unit test for struct -> struct, the append case of nested schema evolution. The existing middle-insert test already covers by-id matching; this pins the "expected 3 got 2" failure mode. * fix(arrow): reconcile nested reorder and rename by field id Always build a PromotePlan for a present column and pass through only when that plan is a no-op. equals_datatype ignored nested names and field ids, so a reorder or same-type rename kept the file layout. Id-less nested children still match by position when their types line up, which covers maps. Nested initial-default stays unsupported (#3261). * fix(arrow): error when only some nested fields have ids The positional fallback stays for a fully id-less struct. If some children have field ids and some do not, null-filling the rest would drop data, so that now errors. A same-type id-less reorder is still positional; it cannot be detected without ids. * fix(arrow): cast id-less nested fields by position Name mapping only stamps top-level ids. When the child counts match, pair by position and let build check each child, so an int still promotes to long. A different count, or ids on only some children, still errors. * test(arrow): compare remaining promote results as whole arrays Use invalid_data! for the PromotePlan DataInvalid errors, matching main. * fix(arrow): accept either list offset width from the file A Parquet Arrow schema hint can make a list column LargeList while the table schema is List. Main cast that column; PromotePlan rejected it. * fix(arrow): apply nested identity constants and drop LargeList targets A missing nested field that is an identity partition source takes that partition value before initial-default or null. schema_to_arrow_schema always emits List, so a LargeList target now errors instead of building a plan; a LargeList file column still promotes to List. The remaining promote tests compare whole arrays. Error tests check ErrorKind, including field ids on only some children. * Simplify PromotePlan and dedup nested promotion tests * fix(arrow): address final review on nested field-id promotion - Add a reader-level test that writes an old nested schema to Parquet and reads it back through an evolved one, pinning that parquet-rs attaches field ids to struct children, list elements, and map keys and values - Reject duplicate source field ids and nested sources read as a non-nested type when the plan is built - Report non-struct map entries as DataInvalid - Pin the id-less same-type reorder fallback with a test until #1845 - Assert map data types, since map array equality skips field names - Point at #3378 where the projection mask skips all-missing structs * test(arrow): cover sliced list promotion with a null slot --------- Co-authored-by: Jordan Epstein --- crates/iceberg/src/arrow/reader/projection.rs | 124 ++ .../src/arrow/record_batch_projector.rs | 14 +- .../src/arrow/record_batch_transformer.rs | 1054 ++++++++++++++++- crates/iceberg/src/arrow/schema.rs | 24 +- 4 files changed, 1152 insertions(+), 64 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index 0f34f00b37..fcb1b2bfd8 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -221,6 +221,8 @@ impl ArrowReader { // Schema evolution: New columns may not exist in old Parquet files. // We only project existing columns; RecordBatchTransformer adds default/NULL values. + // If other leaves are selected, a struct whose requested leaves are all missing gets no + // column and reads as NULL (#3378). let mut indices = vec![]; for field_id in leaf_field_ids { if let Some(col_idx) = column_map.get(field_id) { @@ -2046,4 +2048,126 @@ message schema { .collect(); assert_eq!(ids, vec![2, 3]); } + + /// Reads a file written with an older nested schema through an evolved one. This relies + /// on parquet-rs attaching `PARQUET:field_id` to struct children, list elements, and map + /// keys and values, which the hand-built fixtures in `record_batch_transformer` assume. + #[tokio::test] + async fn test_read_evolved_nested_fields_by_field_id() { + use serde_json::{Value, json}; + + use crate::arrow::schema_to_arrow_schema; + use crate::spec::PrimitiveType::{Int, Long}; + use crate::spec::{ListType, MapType}; + + fn struct_of(fields: &[(i32, &str, PrimitiveType)]) -> Type { + Type::Struct(StructType::new( + fields + .iter() + .map(|(id, name, ty)| { + NestedField::optional(*id, *name, Type::Primitive(ty.clone())).into() + }) + .collect(), + )) + } + fn schema(s: Type, element: Type, value: Type) -> Schema { + let string = Type::Primitive(PrimitiveType::String); + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(Int)).into(), + NestedField::optional(2, "s", s).into(), + NestedField::optional( + 3, + "l", + Type::List(ListType::new( + NestedField::list_element(7, element, true).into(), + )), + ) + .into(), + NestedField::optional( + 4, + "m", + Type::Map(MapType::required(9, string, 10, value)), + ) + .into(), + ]) + .build() + .unwrap() + } + fn to_batch(schema: &Schema, rows: Value) -> RecordBatch { + let fields = schema_to_arrow_schema(schema).unwrap().fields().to_vec(); + serde_arrow::to_record_batch(&fields, &rows).unwrap() + } + + let old_schema = schema( + struct_of(&[(5, "a", Int), (6, "b", PrimitiveType::String)]), + struct_of(&[(8, "x", Int)]), + struct_of(&[(11, "x", Int)]), + ); + // Reorders `s`, adds children before, between, and after existing ones, and + // promotes int to long. + let new_schema = Arc::new(schema( + struct_of(&[ + (6, "b", PrimitiveType::String), + (12, "c", Int), + (5, "a", Long), + ]), + struct_of(&[(13, "y", Int), (8, "x", Int)]), + struct_of(&[(11, "x", Long), (14, "y", Int)]), + )); + let old_batch = to_batch( + &old_schema, + json!([ + {"id": 1, "s": {"a": 1, "b": "p"}, "l": [{"x": 10}, {"x": 11}], "m": {"k1": {"x": 100}}}, + {"id": 2, "s": null, "l": null, "m": {"k2": {"x": 200}}}, + {"id": 3, "s": {"a": 3, "b": null}, "l": [{"x": 30}], "m": null}, + ]), + ); + let expected = to_batch( + &new_schema, + json!([ + { + "id": 1, + "s": {"b": "p", "c": null, "a": 1}, + "l": [{"y": null, "x": 10}, {"y": null, "x": 11}], + "m": {"k1": {"x": 100, "y": null}}, + }, + {"id": 2, "s": null, "l": null, "m": {"k2": {"x": 200, "y": null}}}, + {"id": 3, "s": {"b": null, "c": null, "a": 3}, "l": [{"y": null, "x": 30}], "m": null}, + ]), + ); + + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/old.parquet", tmp_dir.path().to_str().unwrap()); + let mut writer = + ArrowWriter::try_new(File::create(&file_path).unwrap(), old_batch.schema(), None) + .unwrap(); + writer.write(&old_batch).unwrap(); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(new_schema) + .with_project_field_ids(vec![1, 2, 3, 4]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + assert_eq!(result, vec![expected]); + } } diff --git a/crates/iceberg/src/arrow/record_batch_projector.rs b/crates/iceberg/src/arrow/record_batch_projector.rs index 6dda6b0f46..823ef3188c 100644 --- a/crates/iceberg/src/arrow/record_batch_projector.rs +++ b/crates/iceberg/src/arrow/record_batch_projector.rs @@ -20,9 +20,8 @@ use std::sync::Arc; use arrow_array::{Array, ArrayRef, RecordBatch, StructArray, make_array}; use arrow_buffer::NullBuffer; use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; -use parquet::arrow::PARQUET_FIELD_ID_META_KEY; -use crate::arrow::schema::schema_to_arrow_schema; +use crate::arrow::schema::{schema_to_arrow_schema, try_get_field_id_from_metadata}; use crate::error::{Result, invalid_data}; use crate::spec::Schema as IcebergSchema; use crate::{Error, ErrorKind}; @@ -95,16 +94,7 @@ impl RecordBatchProjector { let arrow_schema_with_ids = Arc::new(schema_to_arrow_schema(&iceberg_schema)?); let field_id_fetch_func = |field: &Field| -> Result> { - if let Some(value) = field.metadata().get(PARQUET_FIELD_ID_META_KEY) { - let field_id = value.parse::().map_err(|e| { - invalid_data!("Failed to parse field id") - .with_context("value", value) - .with_source(e) - })?; - Ok(Some(field_id as i64)) - } else { - Ok(None) - } + Ok(try_get_field_id_from_metadata(field)?.map(|id| id as i64)) }; let searchable_field_func = |_field: &Field| -> bool { true }; diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index fe17d336ca..d6d08fd808 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -19,9 +19,10 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use arrow_arith::boolean::is_not_null; +use arrow_array::cast::AsArray; use arrow_array::{ - Array as ArrowArray, ArrayRef, Int32Array, Int64Array, RecordBatch, RecordBatchOptions, - RunArray, StructArray, + Array as ArrowArray, ArrayRef, Int32Array, Int64Array, ListArray, MapArray, RecordBatch, + RecordBatchOptions, RunArray, StructArray, }; use arrow_cast::cast; use arrow_schema::{ @@ -31,6 +32,7 @@ use arrow_schema::{ use arrow_select::zip::zip; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use crate::arrow::schema::try_get_field_id_from_metadata; use crate::arrow::value::{create_primitive_array_repeated, create_primitive_array_single_element}; use crate::arrow::{ datum_to_arrow_type_with_ree, primitive_type_to_arrow_type_with_ree, schema_to_arrow_schema, @@ -145,9 +147,9 @@ pub(crate) enum ColumnSource { // signifies that a column from the file's RecordBatch has undergone // type promotion so the source column with the given index needs - // to be promoted to the specified type + // to be promoted according to the given plan Promote { - target_type: DataType, + plan: PromotePlan, source_index: usize, }, @@ -193,6 +195,285 @@ pub(crate) enum ColumnSource { // post-processing step by using the projection mask. } +/// Reconciles a source array to the target type by matching nested fields on +/// field id rather than position, mirroring iceberg-java's nested readers. +/// +/// The plan is resolved once per file, when the `BatchTransform` is built, so +/// applying it per batch is just index lookups and array assembly. +#[derive(Debug)] +pub(crate) enum PromotePlan { + PassThrough, + Cast(DataType), + Struct { + fields: Fields, + children: Vec, + }, + List { + field: FieldRef, + element: Box, + }, + Map { + field: FieldRef, + entry_fields: Fields, + entries: Vec, + sorted: bool, + }, +} + +#[derive(Debug)] +pub(crate) enum ChildPlan { + FromSource { + source_index: usize, + plan: PromotePlan, + }, + Add { + target_type: DataType, + value: Option, + }, +} + +impl PromotePlan { + fn build( + source: &DataType, + target: &DataType, + snapshot_schema: &IcebergSchema, + constant_fields: &HashMap, + ) -> Result { + if source == target { + return Ok(PromotePlan::PassThrough); + } + match (source, target) { + (DataType::Struct(source_fields), DataType::Struct(target_fields)) => { + Ok(PromotePlan::Struct { + children: Self::build_struct_children( + source_fields, + target_fields, + snapshot_schema, + constant_fields, + )?, + fields: target_fields.clone(), + }) + } + // A file's Arrow schema hint can give either offset width; `apply` converts it. + // Table schemas are always `List` (`ToArrowSchemaConverter::list`). + ( + DataType::List(source_field) | DataType::LargeList(source_field), + DataType::List(target_field), + ) => Ok(PromotePlan::List { + element: Box::new(Self::build( + source_field.data_type(), + target_field.data_type(), + snapshot_schema, + constant_fields, + )?), + field: target_field.clone(), + }), + (DataType::Map(source_entries, _), DataType::Map(target_entries, sorted)) => { + match (source_entries.data_type(), target_entries.data_type()) { + (DataType::Struct(source_fields), DataType::Struct(target_fields)) => { + Ok(PromotePlan::Map { + entries: Self::build_struct_children( + source_fields, + target_fields, + snapshot_schema, + constant_fields, + )?, + entry_fields: target_fields.clone(), + field: target_entries.clone(), + sorted: *sorted, + }) + } + _ => Err(invalid_data!( + "expected struct-typed map entries, got {source_entries:?} and {target_entries:?}" + )), + } + } + (_, DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_)) + | (_, DataType::Map(_, _)) + | (DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_), _) + | (DataType::Map(_, _), _) => { + Err(invalid_data!("cannot promote {source:?} to {target:?}")) + } + _ => Ok(PromotePlan::Cast(target.clone())), + } + } + + fn build_struct_children( + source_fields: &Fields, + target_fields: &Fields, + snapshot_schema: &IcebergSchema, + constant_fields: &HashMap, + ) -> Result> { + let mut source_by_id = HashMap::with_capacity(source_fields.len()); + for (idx, field) in source_fields.iter().enumerate() { + if let Some(id) = try_get_field_id_from_metadata(field)? + && source_by_id.insert(id, idx).is_some() + { + return Err(invalid_data!("duplicate field id {id} in struct")); + } + } + // Name mapping only assigns top-level ids, so fully id-less children match + // by position when the counts line up, and `build` checks each pair. This is + // a compatibility fallback until recursive name mapping (#1845): a same-type + // reorder reads positionally. Any missing id errors instead of nulling that child. + if source_by_id.len() != source_fields.len() { + if source_by_id.is_empty() && source_fields.len() == target_fields.len() { + return source_fields + .iter() + .zip(target_fields.iter()) + .enumerate() + .map(|(source_index, (source_field, target_field))| { + Ok(ChildPlan::FromSource { + source_index, + plan: Self::build( + source_field.data_type(), + target_field.data_type(), + snapshot_schema, + constant_fields, + )?, + }) + }) + .collect(); + } + return Err(invalid_data!( + "cannot reconcile struct fields by id: source fields do not all have field ids" + )); + } + + target_fields + .iter() + .map(|target_field| { + let field_id = try_get_field_id_from_metadata(target_field)?; + match field_id.and_then(|id| source_by_id.get(&id).copied()) { + Some(source_index) => Ok(ChildPlan::FromSource { + plan: Self::build( + source_fields[source_index].data_type(), + target_field.data_type(), + snapshot_schema, + constant_fields, + )?, + source_index, + }), + None => { + // Identity partition constants apply inside structs too. A missing + // child uses that value before initial-default (#3261) or null. + if let Some(ColumnConstant::Scalar(datum)) = + field_id.and_then(|id| constant_fields.get(&id)) + { + return Ok(ChildPlan::Add { + target_type: target_field.data_type().clone(), + value: Some(datum.literal().clone()), + }); + } + // Missing nested fields are null-filled. Applying an + // initial-default here is not implemented yet (#3261). + let iceberg_field = + field_id.and_then(|id| snapshot_schema.field_by_id(id)); + if iceberg_field.is_some_and(|f| f.initial_default.is_some()) { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "initial-default of nested field {} is not supported, see https://github.com/apache/iceberg-rust/issues/3261", + target_field.name() + ), + )); + } + if iceberg_field.map_or(!target_field.is_nullable(), |f| f.required) { + return Err(invalid_data!( + "required nested field {} is absent from the data file and has no initial-default", + target_field.name() + )); + } + Ok(ChildPlan::Add { + target_type: target_field.data_type().clone(), + value: None, + }) + } + } + }) + .collect() + } + + fn apply(&self, array: &ArrayRef) -> Result { + match self { + PromotePlan::PassThrough => Ok(Arc::clone(array)), + PromotePlan::Cast(target_type) => Ok(cast(array.as_ref(), target_type)?), + PromotePlan::Struct { fields, children } => { + let source = array + .as_struct_opt() + .ok_or_else(|| promote_err(array, "struct"))?; + Ok(Arc::new(Self::apply_struct(source, fields, children)?)) + } + PromotePlan::List { field, element } => { + let array = match array.data_type() { + DataType::LargeList(f) => cast(array.as_ref(), &DataType::List(Arc::clone(f)))?, + _ => Arc::clone(array), + }; + let source = array + .as_list_opt::() + .ok_or_else(|| promote_err(&array, "list"))?; + Ok(Arc::new(ListArray::try_new( + field.clone(), + source.offsets().clone(), + element.apply(source.values())?, + source.nulls().cloned(), + )?)) + } + PromotePlan::Map { + field, + entry_fields, + entries, + sorted, + } => { + let source = array + .as_map_opt() + .ok_or_else(|| promote_err(array, "map"))?; + let entries = Self::apply_struct(source.entries(), entry_fields, entries)?; + Ok(Arc::new(MapArray::try_new( + field.clone(), + source.offsets().clone(), + entries, + source.nulls().cloned(), + *sorted, + )?)) + } + } + } + + fn apply_struct( + source: &StructArray, + fields: &Fields, + children: &[ChildPlan], + ) -> Result { + let columns = children + .iter() + .map(|child| match child { + ChildPlan::FromSource { source_index, plan } => { + plan.apply(source.column(*source_index)) + } + ChildPlan::Add { target_type, value } => { + RecordBatchTransformer::create_column(target_type, value.as_ref(), source.len()) + } + }) + .collect::>>()?; + Ok(StructArray::try_new( + fields.clone(), + columns, + source.nulls().cloned(), + )?) + } +} + +fn promote_err(array: &ArrayRef, expected: &str) -> Error { + Error::new( + ErrorKind::Unexpected, + format!( + "expected a {expected} array to promote, got {:?}", + array.data_type() + ), + ) +} + #[derive(Debug)] enum BatchTransform { // Indicates that no changes need to be performed to the RecordBatches @@ -829,25 +1110,26 @@ impl RecordBatchTransformer { // // At this point, all field IDs in the source schema are trustworthy. // No conflict detection needed - schema resolution happened in reader.rs. - let field_by_id = field_id_to_source_schema_map.get(field_id).map( - |(source_field, source_index)| { - if source_field.data_type().equals_datatype(target_type) { - ColumnSource::PassThrough { - source_index: *source_index, - } - } else { - ColumnSource::Promote { - target_type: target_type.clone(), - source_index: *source_index, - } - } - }, - ); - + // // Apply spec's fallback steps for "not present" fields. // Rule #1 (constants) is handled at the beginning of this function - let column_source = if let Some(source) = field_by_id { - source + let column_source = if let Some((source_field, source_index)) = + field_id_to_source_schema_map.get(field_id) + { + match PromotePlan::build( + source_field.data_type(), + target_type, + snapshot_schema, + constant_fields, + )? { + PromotePlan::PassThrough => ColumnSource::PassThrough { + source_index: *source_index, + }, + plan => ColumnSource::Promote { + plan, + source_index: *source_index, + }, + } } else { // Rules #2, #3 and #4: // Rule #2 (name mapping) was already applied in reader.rs if needed. @@ -899,12 +1181,7 @@ impl RecordBatchTransformer { ) -> Result> { let mut field_id_to_source_schema = HashMap::new(); for (source_field_idx, source_field) in source_schema.fields.iter().enumerate() { - // Check if field has a field ID in metadata - if let Some(field_id_str) = source_field.metadata().get(PARQUET_FIELD_ID_META_KEY) { - let this_field_id = field_id_str - .parse() - .map_err(|e| invalid_data!("field id not parseable as an i32: {e}"))?; - + if let Some(this_field_id) = try_get_field_id_from_metadata(source_field)? { field_id_to_source_schema .insert(this_field_id, (source_field.clone(), source_field_idx)); } @@ -930,10 +1207,9 @@ impl RecordBatchTransformer { Ok(match op { ColumnSource::PassThrough { source_index } => columns[*source_index].clone(), - ColumnSource::Promote { - target_type, - source_index, - } => cast(&*columns[*source_index], target_type)?, + ColumnSource::Promote { plan, source_index } => { + plan.apply(&columns[*source_index])? + } ColumnSource::Add { target_type, value } => { Self::create_column(target_type, value.as_ref(), num_rows)? @@ -1147,22 +1423,641 @@ mod test { use std::collections::HashMap; use std::sync::Arc; + use arrow_array::cast::AsArray; + use arrow_array::types::Int32Type; use arrow_array::{ - Array, Date32Array, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, - StringArray, + Array, ArrayRef, Date32Array, Float32Array, Float64Array, Int32Array, Int64Array, + LargeListArray, ListArray, MapArray, RecordBatch, StringArray, StructArray, }; + use arrow_buffer::{NullBuffer, OffsetBuffer}; use arrow_cast::cast; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; use super::{field_with_id, schema_to_arrow_schema}; - use crate::arrow::build_partition_constant; + use crate::ErrorKind; use crate::arrow::record_batch_transformer::{ - RecordBatchTransformer, RecordBatchTransformerBuilder, + PromotePlan, RecordBatchTransformer, RecordBatchTransformerBuilder, }; + use crate::arrow::{DEFAULT_MAP_FIELD_NAME, build_partition_constant}; use crate::spec::{ - ListType, Literal, MapType, NestedField, PrimitiveType, Schema, Struct, Type, + ListType, Literal, MapType, NestedField, PrimitiveType, Schema, Struct, StructType, Type, }; + fn promote(source: &ArrayRef, target: &DataType, schema: &Schema) -> crate::Result { + PromotePlan::build(source.data_type(), target, schema, &HashMap::new())?.apply(source) + } + + fn empty_schema() -> Schema { + Schema::builder().build().unwrap() + } + + fn unevolved_struct_type() -> DataType { + DataType::Struct(Fields::from(vec![field_with_id( + "x", + DataType::Int32, + true, + 5, + )])) + } + + fn evolved_struct_type() -> DataType { + DataType::Struct(Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ])) + } + + fn unevolved_struct_data(x_values: Vec) -> Arc { + Arc::new(StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(x_values)) as ArrayRef], + None, + )) + } + + fn transform_top_level(column: NestedField, file_column: ArrayRef) -> crate::Result { + let name = column.name.clone(); + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + column.into(), + ]) + .build() + .unwrap(), + ); + let mut transformer = RecordBatchTransformerBuilder::new(snapshot_schema, &[1, 2]).build(); + let file_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id(name, file_column.data_type().clone(), true, 2), + ])); + let batch = RecordBatch::try_new(file_schema, vec![ + Arc::new(Int32Array::from(vec![1; file_column.len()])) as ArrayRef, + file_column, + ]) + .unwrap(); + Ok(transformer.process_record_batch(batch)?.column(1).clone()) + } + + #[test] + fn promote_struct_fills_added_fields_by_id() { + let source = Arc::new(StructArray::new( + Fields::from(vec![ + field_with_id("a", DataType::Int32, true, 1), + field_with_id("c", DataType::Utf8, true, 3), + ]), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(StringArray::from(vec!["x", "y"])) as ArrayRef, + ], + None, + )) as ArrayRef; + let fields = Fields::from(vec![ + field_with_id("a", DataType::Int32, true, 1), + field_with_id("b", DataType::Int32, true, 2), + field_with_id("c", DataType::Utf8, true, 3), + field_with_id("d", DataType::Int32, true, 4), + ]); + let target = DataType::Struct(fields.clone()); + + let out = promote(&source, &target, &empty_schema()).unwrap(); + let expected = StructArray::new( + fields, + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None])) as ArrayRef, + Arc::new(StringArray::from(vec!["x", "y"])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None])) as ArrayRef, + ], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_list_element_struct_fills_added_field_by_id() { + let source = Arc::new(ListArray::new( + Arc::new(field_with_id("element", unevolved_struct_type(), true, 4)), + OffsetBuffer::new(vec![0, 1, 2].into()), + unevolved_struct_data(vec![10, 20]), + None, + )) as ArrayRef; + let target = DataType::List(Arc::new(field_with_id( + "element", + evolved_struct_type(), + true, + 4, + ))); + + let out = promote(&source, &target, &empty_schema()).unwrap(); + let values = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None])) as ArrayRef, + ], + None, + ); + let expected = ListArray::new( + Arc::new(field_with_id("element", evolved_struct_type(), true, 4)), + OffsetBuffer::new(vec![0, 1, 2].into()), + Arc::new(values), + None, + ); + assert_eq!(out.as_list::(), &expected); + } + + #[test] + fn promote_sliced_list_with_null_slot() { + let source = ListArray::new( + Arc::new(field_with_id("element", unevolved_struct_type(), true, 4)), + OffsetBuffer::new(vec![0, 1, 1, 3, 4].into()), + unevolved_struct_data(vec![10, 30, 31, 40]), + Some(NullBuffer::from(vec![true, false, true, true])), + ); + let source = Arc::new(source.slice(1, 3)) as ArrayRef; + let target = DataType::List(Arc::new(field_with_id( + "element", + evolved_struct_type(), + true, + 4, + ))); + + let out = promote(&source, &target, &empty_schema()).unwrap(); + let values = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int32Array::from(vec![30, 31, 40])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None, None])) as ArrayRef, + ], + None, + ); + let expected = ListArray::new( + Arc::new(field_with_id("element", evolved_struct_type(), true, 4)), + OffsetBuffer::new(vec![0, 0, 2, 3].into()), + Arc::new(values), + Some(NullBuffer::from(vec![false, true, true])), + ); + assert_eq!(out.as_list::(), &expected); + } + + #[test] + fn promote_map_value_struct_fills_added_field_by_id() { + let entries = StructArray::new( + Fields::from(vec![ + field_with_id("key", DataType::Utf8, false, 7), + field_with_id("value", unevolved_struct_type(), true, 8), + ]), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])) as ArrayRef, + unevolved_struct_data(vec![100, 200]), + ], + None, + ); + let source = Arc::new(MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + OffsetBuffer::new(vec![0, 1, 2].into()), + entries, + None, + false, + )) as ArrayRef; + let target_entries = DataType::Struct(Fields::from(vec![ + field_with_id("key", DataType::Utf8, false, 7), + field_with_id("value", evolved_struct_type(), true, 8), + ])); + let target = DataType::Map( + Arc::new(Field::new("entries", target_entries, false)), + false, + ); + + let out = promote(&source, &target, &empty_schema()).unwrap(); + let values = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int32Array::from(vec![100, 200])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None])) as ArrayRef, + ], + None, + ); + let expected_entries = StructArray::new( + Fields::from(vec![ + field_with_id("key", DataType::Utf8, false, 7), + field_with_id("value", evolved_struct_type(), true, 8), + ]), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])) as ArrayRef, + Arc::new(values) as ArrayRef, + ], + None, + ); + let expected = MapArray::new( + Arc::new(Field::new( + "entries", + expected_entries.data_type().clone(), + false, + )), + OffsetBuffer::new(vec![0, 1, 2].into()), + expected_entries, + None, + false, + ); + assert_eq!(out.data_type(), expected.data_type()); + assert_eq!(out.as_map(), &expected); + } + + #[test] + fn promote_struct_dropped_and_readded_same_name_nulls_by_id() { + let file = StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef], + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(6, "x", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let expected = StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 6)]), + vec![Arc::new(Int32Array::from(vec![None, None])) as ArrayRef], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_struct_without_source_field_ids_errors() { + let source = Arc::new(StructArray::new( + Fields::from(vec![Field::new("x", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef], + None, + )) as ArrayRef; + + let err = promote(&source, &evolved_struct_type(), &empty_schema()).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("do not all have field ids")); + } + + #[test] + fn promote_struct_partial_field_ids_errors() { + let source = Arc::new(StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + Field::new("y", DataType::Int32, true), + ]), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![3, 4])) as ArrayRef, + ], + None, + )) as ArrayRef; + + let err = promote(&source, &evolved_struct_type(), &empty_schema()).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("do not all have field ids")); + } + + #[test] + fn promote_rejects_invalid_source_at_build() { + let duplicate_ids = DataType::Struct(Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 5), + ])); + let map_of = + |entries| DataType::Map(Arc::new(Field::new("entries", entries, false)), false); + for (source, target, message) in [ + (duplicate_ids, evolved_struct_type(), "duplicate field id 5"), + (unevolved_struct_type(), DataType::Int32, "cannot promote"), + ( + map_of(DataType::Int32), + map_of(DataType::Int64), + "expected struct-typed map entries", + ), + ] { + let err = + PromotePlan::build(&source, &target, &empty_schema(), &HashMap::new()).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains(message), "{err}"); + } + } + + #[test] + fn promote_required_nested_field_absent_without_default_errors() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(6, "y", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let source = unevolved_struct_data(vec![1, 2]) as ArrayRef; + + let err = promote(&source, &evolved_struct_type(), &schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("required nested field")); + } + + #[test] + fn promote_nested_field_with_initial_default_errors() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(6, "y", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(42)) + .into(), + ]) + .build() + .unwrap(); + let source = unevolved_struct_data(vec![1, 2]) as ArrayRef; + + let err = promote(&source, &evolved_struct_type(), &schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert!(err.to_string().contains("initial-default")); + } + + #[test] + fn promote_struct_reorders_children_by_id_via_process_record_batch() { + let file = StructArray::new( + Fields::from(vec![ + field_with_id("a", DataType::Int32, true, 5), + field_with_id("b", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![100, 200])) as ArrayRef, + ], + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(6, "b", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(5, "a", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let expected = StructArray::new( + Fields::from(vec![ + field_with_id("b", DataType::Int32, true, 6), + field_with_id("a", DataType::Int32, true, 5), + ]), + vec![ + Arc::new(Int32Array::from(vec![100, 200])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + ], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_struct_renames_child_via_process_record_batch() { + let file = StructArray::new( + Fields::from(vec![field_with_id("x_old", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef], + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(5, "x", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let expected = StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_idless_nested_struct_keeps_data() { + let inner = StructArray::new( + Fields::from(vec![Field::new("x", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef], + None, + ); + let file = StructArray::new( + Fields::from(vec![Field::new("inner", inner.data_type().clone(), true)]), + vec![Arc::new(inner) as ArrayRef], + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional( + 3, + "inner", + Type::Struct(StructType::new(vec![ + NestedField::optional(4, "x", Type::Primitive(PrimitiveType::Long)) + .into(), + ])), + ) + .into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let inner_fields = Fields::from(vec![field_with_id("x", DataType::Int64, true, 4)]); + let expected = StructArray::new( + Fields::from(vec![field_with_id( + "inner", + DataType::Struct(inner_fields.clone()), + true, + 3, + )]), + vec![Arc::new(StructArray::new( + inner_fields, + vec![Arc::new(Int64Array::from(vec![10i64, 20, 30])) as ArrayRef], + None, + )) as ArrayRef], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_idless_map_keeps_data() { + let entries = StructArray::new( + Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int32, true), + ]), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + ], + None, + ); + let map = MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + OffsetBuffer::new(vec![0, 2].into()), + entries, + None, + false, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "m", + Type::Map(MapType::optional( + 3, + Type::Primitive(PrimitiveType::String), + 4, + Type::Primitive(PrimitiveType::Int), + )), + ), + Arc::new(map), + ) + .unwrap(); + let expected_entries = StructArray::new( + Fields::from(vec![ + field_with_id("key", DataType::Utf8, false, 3), + field_with_id("value", DataType::Int32, true, 4), + ]), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + ], + None, + ); + let expected = MapArray::new( + Arc::new(Field::new( + DEFAULT_MAP_FIELD_NAME, + expected_entries.data_type().clone(), + false, + )), + OffsetBuffer::new(vec![0, 2].into()), + expected_entries, + None, + false, + ); + // Map array equality skips the entries field name. + assert_eq!(out.data_type(), expected.data_type()); + assert_eq!(out.as_map(), &expected); + } + + #[test] + fn promote_idless_same_type_reorder_stays_positional() { + // Without ids a same-type reorder is undetectable; #1845 replaces this fallback. + let source = Arc::new(StructArray::new( + Fields::from(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + ]), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + ], + None, + )) as ArrayRef; + let fields = Fields::from(vec![ + field_with_id("b", DataType::Int32, true, 6), + field_with_id("a", DataType::Int32, true, 5), + ]); + + let out = promote(&source, &DataType::Struct(fields.clone()), &empty_schema()).unwrap(); + let expected = StructArray::new( + fields, + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + ], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_large_list_file_column_to_list() { + let element = Arc::new(field_with_id("element", DataType::Int32, true, 3)); + let file = LargeListArray::new( + element.clone(), + OffsetBuffer::new(vec![0i64, 2, 3].into()), + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "l", + Type::List(ListType::new( + NestedField::list_element(3, Type::Primitive(PrimitiveType::Int), false).into(), + )), + ), + Arc::new(file), + ) + .unwrap(); + let expected = ListArray::new( + element, + OffsetBuffer::new(vec![0, 2, 3].into()), + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + None, + ); + assert_eq!(out.as_list::(), &expected); + } + + #[test] + fn promote_evolved_nested_struct_via_process_record_batch() { + // Written before x was promoted to long and renamed from x_old, and before y was added. + let file = StructArray::new( + Fields::from(vec![field_with_id("x_old", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef], + Some(NullBuffer::from(vec![true, true, false])), + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(5, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(6, "y", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let expected = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int64, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int64Array::from(vec![10i64, 20, 30])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None, None])) as ArrayRef, + ], + Some(NullBuffer::from(vec![true, true, false])), + ); + assert_eq!(out.as_struct(), &expected); + } + /// Helper to extract string values from either StringArray or RunEndEncoded /// Returns empty string for null values fn get_string_value(array: &dyn Array, index: usize) -> String { @@ -1174,7 +2069,7 @@ mod test { } } else if let Some(run_array) = array .as_any() - .downcast_ref::>() + .downcast_ref::>() { let values = run_array.values(); let string_values = values @@ -1198,7 +2093,7 @@ mod test { int_array.value(index) } else if let Some(run_array) = array .as_any() - .downcast_ref::>() + .downcast_ref::>() { let values = run_array.values(); let int_values = values @@ -1379,7 +2274,7 @@ mod test { let added_field = NestedField::new( 2, "added_struct", - Type::Struct(crate::spec::StructType::new(vec![ + Type::Struct(StructType::new(vec![ NestedField::optional(3, "child", Type::Primitive(PrimitiveType::Int)) .with_initial_default(Literal::int(42)) .into(), @@ -1415,7 +2310,7 @@ mod test { )); assert_eq!( err.kind(), - crate::ErrorKind::FeatureUnsupported, + ErrorKind::FeatureUnsupported, "required={required}" ); assert!( @@ -1445,7 +2340,7 @@ mod test { NestedField::optional( 3, "struct_col", - Type::Struct(crate::spec::StructType::new(vec![ + Type::Struct(StructType::new(vec![ NestedField::optional( 100, "inner_field", @@ -1500,7 +2395,7 @@ mod test { let struct_column = result .column(2) .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); assert!(struct_column.is_null(0)); assert!(struct_column.is_null(1)); @@ -1550,7 +2445,7 @@ mod test { NestedField::optional( 7, "s", - Type::Struct(crate::spec::StructType::new(vec![ + Type::Struct(StructType::new(vec![ NestedField::optional(8, "a", Type::Primitive(PrimitiveType::String)) .into(), NestedField::optional( @@ -2075,6 +2970,77 @@ mod test { assert_eq!(get_string_value(result.column(2).as_ref(), 1), "Bob"); } + /// `identity(s.region)` with `region` absent from the file struct. The constant + /// is stored on field id 5; the struct promote has to read it. + #[test] + fn identity_partition_fills_missing_nested_field() { + use crate::spec::{Struct, Transform}; + + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional( + 5, + "region", + Type::Primitive(PrimitiveType::String), + ) + .into(), + NestedField::optional(6, "x", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(0) + .add_partition_field("s.region", "region", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let partition_data = Struct::from_iter(vec![Some(Literal::string("us"))]); + + let file = StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 6)]), + vec![Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef], + None, + ); + let parquet_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "s", + file.data_type().clone(), + true, + 2, + )])); + let mut transformer = RecordBatchTransformerBuilder::new(snapshot_schema, &[2]) + .with_partition(partition_spec, partition_data) + .unwrap() + .build(); + let batch = RecordBatch::try_new(parquet_schema, vec![Arc::new(file) as ArrayRef]).unwrap(); + let result = transformer.process_record_batch(batch).unwrap(); + + let expected = StructArray::new( + Fields::from(vec![ + field_with_id("region", DataType::Utf8, true, 5), + field_with_id("x", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(StringArray::from(vec!["us", "us"])) as ArrayRef, + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + ], + None, + ); + assert_eq!(result.column(0).as_struct(), &expected); + } + /// Test that a dropped identity partition source column is skipped rather than erroring. /// /// A historical spec may use `identity(col)` where `col` was later dropped from the diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index e88c24d1a7..338f1776d1 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -317,15 +317,23 @@ pub fn arrow_type_to_type(ty: &DataType) -> Result { const ARROW_FIELD_DOC_KEY: &str = "doc"; +pub(super) fn try_get_field_id_from_metadata(field: &Field) -> Result> { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .map(|value| { + value.parse::().map_err(|e| { + invalid_data!("Failed to parse field id") + .with_context("value", value) + .with_source(e) + }) + }) + .transpose() +} + pub(super) fn get_field_id_from_metadata(field: &FieldRef) -> Result { - if let Some(value) = field.metadata().get(PARQUET_FIELD_ID_META_KEY) { - return value.parse::().map_err(|e| { - invalid_data!("Failed to parse field id") - .with_context("value", value) - .with_source(e) - }); - } - Err(invalid_data!("Field id not found in metadata")) + try_get_field_id_from_metadata(field)? + .ok_or_else(|| invalid_data!("Field id not found in metadata")) } fn get_field_doc(field: &FieldRef) -> Option { From 2fddb1601a9af1c97a559baa9865e1494d93756c Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 11 Oct 2026 18:19:26 +0000 Subject: [PATCH 55/63] fix(spec): reject boolean literals for non-boolean types (#3334) * fix(spec): reject boolean literals for non-boolean types * test(spec): polish boolean literal validation coverage --- crates/iceberg/src/spec/values/serde.rs | 5 +- crates/iceberg/src/spec/values/tests.rs | 150 +++++++++++++++++++++++- 2 files changed, 153 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/spec/values/serde.rs b/crates/iceberg/src/spec/values/serde.rs index a9c6115171..7fa41159d3 100644 --- a/crates/iceberg/src/spec/values/serde.rs +++ b/crates/iceberg/src/spec/values/serde.rs @@ -428,7 +428,10 @@ pub(crate) mod _serde { }; match self { RawLiteralEnum::Null => Ok(None), - RawLiteralEnum::Boolean(v) => Ok(Some(Literal::bool(v))), + RawLiteralEnum::Boolean(v) => match ty { + Type::Primitive(PrimitiveType::Boolean) => Ok(Some(Literal::bool(v))), + _ => Err(invalid_err("boolean")), + }, RawLiteralEnum::Int(v) => match ty { Type::Primitive(PrimitiveType::Int) => Ok(Some(Literal::int(v))), Type::Primitive(PrimitiveType::Long) => Ok(Some(Literal::long(i64::from(v)))), diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index 61f83efee1..63ff9f636b 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -31,7 +31,9 @@ use crate::ErrorKind; use crate::avro::schema_to_avro_schema; use crate::spec::Schema; use crate::spec::Type::Primitive; -use crate::spec::datatypes::{ListType, MapType, NestedField, PrimitiveType, StructType, Type}; +use crate::spec::datatypes::{ + ListType, MapType, NestedField, PrimitiveType, StructType, Type, VariantType, +}; use crate::spec::values::datum::{INT_MAX, INT_MIN, LONG_MAX, LONG_MIN}; use crate::spec::values::{Datum, Literal, Map, PrimitiveLiteral, RawLiteral, Struct}; @@ -820,6 +822,152 @@ fn test_raw_literal_bytes_unsupported_type() { check_raw_literal_bytes_error_via_avro(bytes, &Primitive(PrimitiveType::Int)); } +#[test] +fn test_raw_literal_boolean_round_trip() { + for value in [false, true] { + let ty = Primitive(PrimitiveType::Boolean); + check_raw_literal_json_serde(Literal::bool(value), &ty); + check_convert_with_avro(Literal::bool(value), &ty); + + for json in [ + serde_json::json!({"type": "boolean", "literal": value}), + serde_json::json!(["boolean", value]), + ] { + assert_eq!( + serde_json::from_value::(json).unwrap(), + Datum::bool(value) + ); + } + } +} + +#[test] +fn test_datum_json_boolean_rejects_type_mismatch() { + for ty in ["int", "long", "string", "unknown"] { + for value in [false, true] { + for json in [ + serde_json::json!({"type": ty, "literal": value}), + serde_json::json!([ty, value]), + ] { + let error = serde_json::from_value::(json).unwrap_err(); + assert!(error.to_string().contains("raw literal (boolean)")); + assert!(error.to_string().contains("type mismatch")); + } + } + } +} + +#[test] +fn test_raw_literal_null_for_boolean_type() { + let ty = Primitive(PrimitiveType::Boolean); + let avro_literal: RawLiteral = apache_avro::from_value(&Value::Null).unwrap(); + let json_literal: RawLiteral = serde_json::from_str("null").unwrap(); + + for literal in [avro_literal, json_literal] { + assert_eq!(literal.try_into(&ty).unwrap(), None); + } +} + +#[test] +fn test_raw_literal_boolean_rejects_type_mismatch() { + let types = [ + Primitive(PrimitiveType::Int), + Primitive(PrimitiveType::Long), + Primitive(PrimitiveType::Float), + Primitive(PrimitiveType::Double), + Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }), + Primitive(PrimitiveType::Date), + Primitive(PrimitiveType::Time), + Primitive(PrimitiveType::Timestamp), + Primitive(PrimitiveType::Timestamptz), + Primitive(PrimitiveType::TimestampNs), + Primitive(PrimitiveType::TimestamptzNs), + Primitive(PrimitiveType::String), + Primitive(PrimitiveType::Uuid), + Primitive(PrimitiveType::Fixed(1)), + Primitive(PrimitiveType::Binary), + Primitive(PrimitiveType::Unknown), + Type::Struct(StructType::new(vec![])), + Type::List(ListType::new( + NestedField::list_element(1, Primitive(PrimitiveType::Boolean), false).into(), + )), + Type::Map(MapType::new( + NestedField::map_key_element(1, Primitive(PrimitiveType::String)).into(), + NestedField::map_value_element(2, Primitive(PrimitiveType::Boolean), false).into(), + )), + Type::Variant(VariantType), + ]; + + for value in [false, true] { + for ty in &types { + let avro_literal: RawLiteral = apache_avro::from_value(&Value::Boolean(value)).unwrap(); + let json_literal: RawLiteral = + serde_json::from_value(serde_json::json!(value)).unwrap(); + + for literal in [avro_literal, json_literal] { + let error = literal.try_into(ty).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.to_string().contains("raw literal (boolean)")); + assert!(error.to_string().contains("type mismatch")); + } + } + } +} + +#[test] +fn test_raw_literal_boolean_rejects_nested_type_mismatch() { + let cases = [ + ( + serde_json::json!({"col": true}), + Value::Record(vec![("col".to_string(), Value::Boolean(true))]), + Type::Struct(StructType::new(vec![ + NestedField::required(1, "col", Primitive(PrimitiveType::Int)).into(), + ])), + ), + ( + serde_json::json!([false]), + Value::Array(vec![Value::Boolean(false)]), + Type::List(ListType::new( + NestedField::list_element(1, Primitive(PrimitiveType::Int), false).into(), + )), + ), + ( + serde_json::json!({"key": true}), + Value::Map([("key".to_string(), Value::Boolean(true))].into()), + Type::Map(MapType::new( + NestedField::map_key_element(1, Primitive(PrimitiveType::String)).into(), + NestedField::map_value_element(2, Primitive(PrimitiveType::Int), false).into(), + )), + ), + ( + serde_json::json!([{"key": false, "value": 1}]), + Value::Array(vec![Value::Record(vec![ + ("key".to_string(), Value::Boolean(false)), + ("value".to_string(), Value::Int(1)), + ])]), + Type::Map(MapType::new( + NestedField::map_key_element(1, Primitive(PrimitiveType::Int)).into(), + NestedField::map_value_element(2, Primitive(PrimitiveType::Int), false).into(), + )), + ), + ]; + + for (json_value, avro_value, ty) in cases { + let avro_literal: RawLiteral = apache_avro::from_value(&avro_value).unwrap(); + let json_literal: RawLiteral = serde_json::from_value(json_value).unwrap(); + + for literal in [avro_literal, json_literal] { + let error = literal.try_into(&ty).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.to_string().contains("raw literal (boolean)")); + assert!(error.to_string().contains("type mismatch")); + } + } +} + #[test] fn avro_convert_test_int() { check_convert_with_avro( From 9092073f988b3e6401d4ca00e2b50b7a53434ee1 Mon Sep 17 00:00:00 2001 From: NoahKusaba <97474796+NoahKusaba@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:20:15 +0000 Subject: [PATCH 56/63] fix(arrow): strip metadata from nested fields and normalize container field names (#3303) * fix(arrow): strip metadata from list elements and map entries strip_metadata_from_schema failed with "Field stack underflow in list" on any schema with a list or map column. MetadataStripVisitor pushed a field onto its stack only in before_field, which the visitor does not call for list elements, map keys or map values, so rebuilding those popped from an empty stack. Push in before_list_element, before_map_key and before_map_value too. The stack now holds each field's name and nullability rather than a Field with a placeholder type, and one pop_field helper rebuilds fields for struct, list, map and primitive. Document the two normalizations the function makes besides removing metadata, which callers comparing stripped schemas rely on: a map's entries field is renamed to DEFAULT_MAP_FIELD_NAME, and a dictionary-encoded field becomes its value type. Closes #3297 * test(arrow): strip metadata from fields with repeated nested names * refactor(arrow): rename MetadataStripVisitor's field stack helpers to start_field and finish_field * docs(arrow): clarify visitor field hooks and strip_metadata_from_schema naming before_field and after_field run for every schema and struct field, not for list elements or map keys and values. strip_metadata_from_schema keeps list element names, and renames only a map's entries field. * normalize list and map field names when stripping metadata --- crates/iceberg/src/arrow/schema.rs | 388 ++++++++++++++++++++++++----- 1 file changed, 325 insertions(+), 63 deletions(-) diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 338f1776d1..996ced09ef 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -37,8 +37,9 @@ use uuid::Uuid; use crate::error::{Result, invalid_data}; use crate::spec::decimal_utils::i128_from_be_bytes; use crate::spec::{ - Datum, FIRST_FIELD_ID, ListType, MapType, NestedField, NestedFieldRef, PrimitiveLiteral, - PrimitiveType, Schema, SchemaVisitor, StructType, Type, VariantType, + Datum, FIRST_FIELD_ID, LIST_FIELD_NAME, ListType, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, + MapType, NestedField, NestedFieldRef, PrimitiveLiteral, PrimitiveType, Schema, SchemaVisitor, + StructType, Type, VariantType, }; use crate::{Error, ErrorKind}; @@ -110,12 +111,16 @@ pub trait ArrowSchemaVisitor { /// Return type of this visitor on arrow schema. type U; - /// Called before struct/list/map field. + /// Called before each field of the schema or of a struct, whatever its type. + /// List elements and map keys and values use [`Self::before_list_element`], + /// [`Self::before_map_key`] and [`Self::before_map_value`] instead. fn before_field(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } - /// Called after struct/list/map field. + /// Called after each field of the schema or of a struct, whatever its type. + /// List elements and map keys and values use [`Self::after_list_element`], + /// [`Self::after_map_key`] and [`Self::after_map_value`] instead. fn after_field(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } @@ -1198,16 +1203,43 @@ pub(crate) fn primitive_type_to_arrow_type_with_ree(primitive_type: &PrimitiveTy /// including nested struct, list, and map fields. This is useful for schema comparison /// where metadata differences should be ignored. struct MetadataStripVisitor { - /// Stack to track field information during traversal - field_stack: Vec, + /// Fields whose types are still being visited, innermost last + pending_fields: Vec, +} + +/// What `MetadataStripVisitor` keeps of a field until its type is visited +struct PendingField { + name: String, + nullable: bool, } impl MetadataStripVisitor { fn new() -> Self { Self { - field_stack: Vec::new(), + pending_fields: Vec::new(), } } + + /// Records a field whose type is about to be visited. + fn start_field(&mut self, name: &str, nullable: bool) -> Result<()> { + self.pending_fields.push(PendingField { + name: name.to_string(), + nullable, + }); + Ok(()) + } + + /// Builds the innermost pending field with `data_type` and no metadata. + /// `kind` names the type in the error for an unbalanced stack. + fn finish_field(&mut self, data_type: DataType, kind: &str) -> Result { + let PendingField { name, nullable } = self.pending_fields.pop().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!("Field stack underflow in {kind}"), + ) + })?; + Ok(Field::new(name, data_type, nullable)) + } } impl ArrowSchemaVisitor for MetadataStripVisitor { @@ -1215,46 +1247,40 @@ impl ArrowSchemaVisitor for MetadataStripVisitor { type U = ArrowSchema; fn before_field(&mut self, field: &FieldRef) -> Result<()> { - // Store field name and nullability for later reconstruction - self.field_stack.push(Field::new( - field.name(), - DataType::Null, // Placeholder, will be replaced - field.is_nullable(), - )); - Ok(()) + self.start_field(field.name(), field.is_nullable()) } - fn after_field(&mut self, _field: &FieldRef) -> Result<()> { - Ok(()) + fn before_list_element(&mut self, field: &FieldRef) -> Result<()> { + self.start_field(LIST_FIELD_NAME, field.is_nullable()) + } + + fn before_map_key(&mut self, _field: &FieldRef) -> Result<()> { + // Map keys are always required, as `schema_to_arrow_schema` builds them. + self.start_field(MAP_KEY_FIELD_NAME, false) + } + + fn before_map_value(&mut self, field: &FieldRef) -> Result<()> { + self.start_field(MAP_VALUE_FIELD_NAME, field.is_nullable()) } fn schema(&mut self, _schema: &ArrowSchema, values: Vec) -> Result { + if !self.pending_fields.is_empty() { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "{} fields left on the field stack after traversal", + self.pending_fields.len() + ), + )); + } Ok(ArrowSchema::new(values)) } fn r#struct(&mut self, _fields: &Fields, results: Vec) -> Result { - // Pop the struct field from the stack - let field_info = self - .field_stack - .pop() - .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Field stack underflow in struct"))?; - - // Reconstruct struct field without metadata - Ok(Field::new( - field_info.name(), - DataType::Struct(Fields::from(results)), - field_info.is_nullable(), - )) + self.finish_field(DataType::Struct(Fields::from(results)), "struct") } fn list(&mut self, list: &DataType, value: Self::T) -> Result { - // Pop the list field from the stack - let field_info = self - .field_stack - .pop() - .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Field stack underflow in list"))?; - - // Reconstruct list field without metadata let list_type = match list { DataType::List(_) => DataType::List(Arc::new(value)), DataType::LargeList(_) => DataType::LargeList(Arc::new(value)), @@ -1267,20 +1293,10 @@ impl ArrowSchemaVisitor for MetadataStripVisitor { } }; - Ok(Field::new( - field_info.name(), - list_type, - field_info.is_nullable(), - )) + self.finish_field(list_type, "list") } fn map(&mut self, map: &DataType, key_value: Self::T, value: Self::T) -> Result { - // Pop the map field from the stack - let field_info = self - .field_stack - .pop() - .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Field stack underflow in map"))?; - // Reconstruct the map's struct field (contains key and value) let struct_field = Field::new( DEFAULT_MAP_FIELD_NAME, @@ -1299,26 +1315,11 @@ impl ArrowSchemaVisitor for MetadataStripVisitor { } }; - // Reconstruct map field without metadata - Ok(Field::new( - field_info.name(), - DataType::Map(Arc::new(struct_field), sorted), - field_info.is_nullable(), - )) + self.finish_field(DataType::Map(Arc::new(struct_field), sorted), "map") } fn primitive(&mut self, p: &DataType) -> Result { - // Pop the primitive field from the stack - let field_info = self.field_stack.pop().ok_or_else(|| { - Error::new(ErrorKind::Unexpected, "Field stack underflow in primitive") - })?; - - // Return field without metadata - Ok(Field::new( - field_info.name(), - p.clone(), - field_info.is_nullable(), - )) + self.finish_field(p.clone(), "primitive") } } @@ -1328,6 +1329,21 @@ impl ArrowSchemaVisitor for MetadataStripVisitor { /// including nested struct, list, and map fields. This is useful for schema comparison /// where metadata differences should be ignored. /// +/// Some other differences are normalized away too, so that schemas that differ only in +/// them compare equal once stripped. The fields inside lists and maps are renamed as +/// [`schema_to_arrow_schema`] names them, and dictionaries are unwrapped: +/// - A list's element field is renamed to [`LIST_FIELD_NAME`], so `item` (the arrow-rs +/// default) becomes `element`. +/// - A map's entries field is renamed to [`DEFAULT_MAP_FIELD_NAME`] and made non-nullable. +/// - A map's key and value fields are renamed to [`MAP_KEY_FIELD_NAME`] and +/// [`MAP_VALUE_FIELD_NAME`], so `keys` and `values` (the arrow-rs `MapBuilder` default) +/// become `key` and `value`, and the key is made non-nullable. +/// - A dictionary-encoded field becomes a field of the dictionary's value type. +/// +/// The names of the schema's fields and of struct fields are kept. Extension type +/// metadata (`ARROW:extension:*`) is dropped with the rest, so a variant field becomes a +/// plain struct of its `metadata` and `value` fields. +/// /// # Arguments /// * `schema` - The Arrow schema to strip metadata from /// @@ -2608,4 +2624,250 @@ mod tests { pretty_assertions::assert_eq!(schema, expected); assert_eq!(schema.highest_field_id(), 17); } + + /// Every field is rebuilt without metadata, at any depth and inside any + /// container. Fields are rebuilt in nesting order rather than looked up by + /// name, so a path that repeats names (`a.b.a.b`) keeps each level's name + /// and nullability. + #[test] + fn test_strip_metadata_from_nested_schema() { + // The same schema, built with and without metadata on the schema and every field. + let schema = |with_metadata: bool| { + let metadata = |value: &str| { + if with_metadata { + HashMap::from([("k".to_string(), value.to_string())]) + } else { + HashMap::new() + } + }; + let field = |name: &str, ty: DataType, nullable: bool| { + Field::new(name, ty, nullable).with_metadata(metadata(name)) + }; + let element = |ty: DataType| Arc::new(field(LIST_FIELD_NAME, ty, false)); + let ints = || DataType::List(element(DataType::Int32)); + let map = |value: DataType, sorted: bool| { + let entries = Fields::from(vec![ + field(MAP_KEY_FIELD_NAME, DataType::Utf8, false), + field(MAP_VALUE_FIELD_NAME, value, true), + ]); + let entries = field(DEFAULT_MAP_FIELD_NAME, DataType::Struct(entries), false); + DataType::Map(Arc::new(entries), sorted) + }; + let struct_of = |fields: Vec| DataType::Struct(Fields::from(fields)); + + let a_b_a_b = struct_of(vec![field( + "b", + struct_of(vec![field( + "a", + struct_of(vec![field("b", DataType::Int32, true)]), + false, + )]), + false, + )]); + ArrowSchema::new(vec![ + field("list", ints(), true), + field( + "large_list", + DataType::LargeList(element(DataType::Int32)), + false, + ), + field( + "fixed_list", + DataType::FixedSizeList(element(DataType::Int32), 2), + true, + ), + field("list_of_list", DataType::List(element(ints())), false), + field("sorted_map", map(ints(), true), false), + field( + "map_of_struct", + map( + struct_of(vec![ + field("a", DataType::Int32, true), + field("b", ints(), false), + ]), + false, + ), + true, + ), + field( + "list_of_struct", + DataType::List(element(struct_of(vec![field( + "m", + map(ints(), false), + true, + )]))), + true, + ), + field("a", a_b_a_b, true), + ]) + .with_metadata(metadata("schema")) + }; + + let (input, expected) = (schema(true), schema(false)); + // Otherwise a builder that stopped adding metadata would pass trivially. + assert_ne!(input, expected); + pretty_assertions::assert_eq!(strip_metadata_from_schema(&input).unwrap(), expected); + } + + /// arrow-rs names a list's element `item`, and an arrow-rs `MapBuilder` names + /// a map's fields `entries`, `keys` and `values`. Once stripped, a schema + /// using those names, with a nullable map key, compares equal to the same + /// schema from `schema_to_arrow_schema`. + #[test] + fn test_strip_metadata_normalizes_container_field_names() { + // A list of ints and a map from string to a list of ints, with the given + // names for the list elements and the map's entries, key and value. + let schema = |[element, entries, key, value]: [&str; 4], key_nullable: bool| { + let ints = + |nullable| DataType::List(Arc::new(Field::new(element, DataType::Int32, nullable))); + let key_value = Fields::from(vec![ + Field::new(key, DataType::Utf8, key_nullable), + Field::new(value, ints(false), true), + ]); + let entries = Field::new(entries, DataType::Struct(key_value), false); + ArrowSchema::new(vec![ + Field::new("l", ints(true), true), + Field::new("m", DataType::Map(Arc::new(entries), false), true), + ]) + }; + let arrow_rs = schema(["item", "entries", "keys", "values"], true); + let expected = schema( + [ + LIST_FIELD_NAME, + DEFAULT_MAP_FIELD_NAME, + MAP_KEY_FIELD_NAME, + MAP_VALUE_FIELD_NAME, + ], + false, + ); + + // The same schema as an Iceberg table's. + let int_list = |id, required| { + let int = Type::Primitive(PrimitiveType::Int); + Type::List(ListType::new( + NestedField::list_element(id, int, required).into(), + )) + }; + let string_to_int_list = MapType::new( + NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String)).into(), + NestedField::map_value_element(5, int_list(6, true), false).into(), + ); + let iceberg = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "l", int_list(2, false)).into(), + NestedField::optional(3, "m", Type::Map(string_to_int_list)).into(), + ]) + .build() + .unwrap(); + + pretty_assertions::assert_eq!(strip_metadata_from_schema(&arrow_rs).unwrap(), expected); + pretty_assertions::assert_eq!( + strip_metadata_from_schema(&schema_to_arrow_schema(&iceberg).unwrap()).unwrap(), + expected + ); + } + + /// Names and nullability that `schema_to_arrow_schema` doesn't fix, such as a + /// struct field's name or a list element's nullability, still tell schemas apart. + #[test] + fn test_strip_metadata_keeps_other_differences() { + let schema = |child: &str, element_nullable: bool| { + let child = Field::new(child, DataType::Int32, true); + let element = Field::new("item", DataType::Int32, element_nullable); + ArrowSchema::new(vec![ + Field::new("s", DataType::Struct(Fields::from(vec![child])), true), + Field::new("l", DataType::List(Arc::new(element)), true), + ]) + }; + let strip = |schema: ArrowSchema| strip_metadata_from_schema(&schema).unwrap(); + + assert_ne!(strip(schema("a", true)), strip(schema("b", true))); + assert_ne!(strip(schema("a", true)), strip(schema("a", false))); + } + + /// Stripping drops the `arrow.parquet.variant` extension type with the rest of + /// the metadata, so a variant becomes a plain struct, in a list or map too. + #[test] + fn test_strip_metadata_turns_variants_into_structs() { + let variant = || Type::Variant(VariantType); + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "v", variant()).into(), + NestedField::optional( + 2, + "l", + Type::List(ListType::new( + NestedField::list_element(3, variant(), false).into(), + )), + ) + .into(), + NestedField::optional( + 4, + "m", + Type::Map(MapType::new( + NestedField::map_key_element(5, Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::map_value_element(6, variant(), false).into(), + )), + ) + .into(), + ]) + .build() + .unwrap(); + + let storage = || { + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + ])) + }; + let entries = Fields::from(vec![ + Field::new(MAP_KEY_FIELD_NAME, DataType::Utf8, false), + Field::new(MAP_VALUE_FIELD_NAME, storage(), true), + ]); + let entries = Field::new(DEFAULT_MAP_FIELD_NAME, DataType::Struct(entries), false); + let expected = ArrowSchema::new(vec![ + Field::new("v", storage(), true), + Field::new( + "l", + DataType::List(Arc::new(Field::new(LIST_FIELD_NAME, storage(), true))), + true, + ), + Field::new("m", DataType::Map(Arc::new(entries), false), true), + ]); + + pretty_assertions::assert_eq!( + strip_metadata_from_schema(&schema_to_arrow_schema(&schema).unwrap()).unwrap(), + expected + ); + } + + /// A dictionary-encoded field compares equal to a plain field of its value + /// type once stripped, at any depth. + #[test] + fn test_strip_metadata_unwraps_dictionaries() { + let schema = |dictionary_encoded: bool| { + let maybe_dictionary = |key: DataType, value: DataType| { + if dictionary_encoded { + DataType::Dictionary(Box::new(key), Box::new(value)) + } else { + value + } + }; + let element = Field::new( + LIST_FIELD_NAME, + maybe_dictionary(DataType::Int8, DataType::LargeUtf8), + false, + ); + ArrowSchema::new(vec![ + Field::new("d", maybe_dictionary(DataType::Int32, DataType::Utf8), true), + Field::new("l", DataType::List(Arc::new(element)), true), + ]) + }; + + pretty_assertions::assert_eq!( + strip_metadata_from_schema(&schema(true)).unwrap(), + schema(false) + ); + } } From d3e9f456e29b10ba95f11b3a2bc5c9dc546e1d93 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Sun, 11 Oct 2026 18:28:33 +0000 Subject: [PATCH 57/63] perf(reader): read leaf field IDs from metadata in the projection mask (#3396) get_arrow_projection_mask_with_field_ids carried each leaf's field ID from its first pass to its second through a HashMap. Hashing an Arrow Field hashes its whole DataType and allocates a sorted copy of its metadata keys, and the map grew from empty. Read the ID from the leaf's PARQUET:field_id metadata in both passes instead. Closes #3394 --- crates/iceberg/src/arrow/reader/projection.rs | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index fcb1b2bfd8..8f2afe79e4 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -178,23 +178,16 @@ impl ArrowReader { // Pre-project only the fields that have been selected, possibly avoiding converting // some Arrow types that are not yet supported. - let mut projected_fields: HashMap = HashMap::new(); let projected_arrow_schema = ArrowSchema::new_with_metadata( fields.filter_leaves(|_, f| { - f.metadata() - .get(PARQUET_FIELD_ID_META_KEY) - .and_then(|field_id| i32::from_str(field_id).ok()) - .is_some_and(|field_id| { - projected_fields.insert((*f).clone(), field_id); - leaf_field_id_set.contains(&field_id) - }) + parquet_field_id(f).is_some_and(|field_id| leaf_field_id_set.contains(&field_id)) }), arrow_schema.metadata().clone(), ); let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?; fields.filter_leaves(|idx, field| { - let Some(field_id) = projected_fields.get(field).cloned() else { + let Some(field_id) = parquet_field_id(field) else { return false; }; @@ -381,11 +374,7 @@ pub(super) fn build_fallback_field_id_map( fn build_field_id_map_from_arrow_schema(arrow_schema: &ArrowSchemaRef) -> HashMap { let mut column_map = HashMap::new(); arrow_schema.fields().filter_leaves(|idx, field| { - if let Some(field_id) = field - .metadata() - .get(PARQUET_FIELD_ID_META_KEY) - .and_then(|value| i32::from_str(value).ok()) - { + if let Some(field_id) = parquet_field_id(field) { column_map.insert(field_id, idx); } false @@ -393,6 +382,14 @@ fn build_field_id_map_from_arrow_schema(arrow_schema: &ArrowSchemaRef) -> HashMa column_map } +/// Returns the field ID in `field`'s `PARQUET:field_id` metadata, if it has a valid one. +fn parquet_field_id(field: &Field) -> Option { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .and_then(|field_id| i32::from_str(field_id).ok()) +} + /// Apply name mapping to Arrow schema for Parquet files lacking field IDs. /// /// Assigns Iceberg field IDs based on column names using the name mapping, From acd610997de78f986ece9f89c3bf476de5e1f5a5 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 11 Oct 2026 18:29:22 +0000 Subject: [PATCH 58/63] perf(reader): speed up row group filtering and name mapping (#3269) * perf(reader): optimize row group filter intersection * perf(reader): add row group intersection benchmark * style: format iceberg benchmark manifest * perf(reader): avoid allocations in name mapping lookup * perf(reader): prune row groups from split candidates * fix(reader): format tests and preallocate name mapping index * fix(reader): tighten candidate pruning and name mapping coverage --- crates/iceberg/src/arrow/reader/pipeline.rs | 53 +++++----- crates/iceberg/src/arrow/reader/projection.rs | 49 ++++++++-- crates/iceberg/src/arrow/reader/row_filter.rs | 96 ++++++++++++++++++- 3 files changed, 161 insertions(+), 37 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 18ba42d3b9..84b4467d37 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -20,6 +20,7 @@ //! predicates, row-group / row selection, and delete handling into a stream //! of transformed Arrow `RecordBatch`es. +use std::borrow::Cow; use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::AtomicU64; @@ -133,6 +134,17 @@ struct FileScanTaskReader { scan_metrics: ScanMetrics, } +// Borrow an existing selection, or enumerate all groups for a whole-file scan. +fn candidate_row_groups( + selected_row_groups: Option<&[usize]>, + num_row_groups: usize, +) -> Cow<'_, [usize]> { + match selected_row_groups { + Some(indices) => Cow::Borrowed(indices), + None => Cow::Owned((0..num_row_groups).collect()), + } +} + impl FileScanTaskReader { async fn process(self, task: FileScanTask) -> Result { let should_load_page_index = (self.row_selection_enabled && task.predicate().is_some()) @@ -525,48 +537,35 @@ impl FileScanTaskReader { record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter); if self.row_group_filtering_enabled { - let predicate_filtered_row_groups = ArrowReader::get_selected_row_group_indices( + let candidates = candidate_row_groups( + selected_row_group_indices.as_deref(), + record_batch_stream_builder.metadata().num_row_groups(), + ); + + selected_row_group_indices = Some(ArrowReader::get_selected_row_group_indices( &predicate, record_batch_stream_builder.metadata(), &field_id_map, task.schema(), - )?; - - // Merge predicate-based filtering with byte range filtering (if present) - // by taking the intersection of both filters - selected_row_group_indices = match selected_row_group_indices { - Some(byte_range_filtered) => { - // Keep only row groups that are in both filters - let intersection: Vec = byte_range_filtered - .into_iter() - .filter(|idx| predicate_filtered_row_groups.contains(idx)) - .collect(); - Some(intersection) - } - None => Some(predicate_filtered_row_groups), - }; + candidates.as_ref(), + )?); } if self.bloom_filter_enabled { - let all_rgs; - let candidate_rgs = match &selected_row_group_indices { - Some(indices) => indices.as_slice(), - None => { - all_rgs = (0..record_batch_stream_builder.metadata().num_row_groups()) - .collect::>(); - &all_rgs - } - }; + let candidates = candidate_row_groups( + selected_row_group_indices.as_deref(), + record_batch_stream_builder.metadata().num_row_groups(), + ); let bloom_filtered = Self::filter_row_groups_by_bloom_filter( &predicate, &mut record_batch_stream_builder, - candidate_rgs, + candidates.as_ref(), &field_id_map, ) .await?; - if bloom_filtered.len() < candidate_rgs.len() { + if bloom_filtered.len() < candidates.len() { selected_row_group_indices = Some(bloom_filtered); } } diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index 8f2afe79e4..2fa6bf18f9 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -421,6 +421,16 @@ pub(super) fn apply_name_mapping_to_arrow_schema( "Schema already has field IDs - name mapping should not be applied" ); + let mut field_ids_by_name = HashMap::with_capacity(name_mapping.fields().len()); + for mapped_field in name_mapping.fields() { + for name in mapped_field.names() { + // Preserve the existing first-match behavior for duplicate aliases. + field_ids_by_name + .entry(name.as_str()) + .or_insert(mapped_field.field_id()); + } + } + let fields_with_mapped_ids: Vec<_> = arrow_schema .fields() .iter() @@ -432,15 +442,12 @@ pub(super) fn apply_name_mapping_to_arrow_schema( // If the field isn't in the mapping, leave it WITHOUT assigning an ID // (matching Java's behavior of returning the field unchanged). // Later, during projection, fields without IDs are filtered out. - let mapped_field_opt = name_mapping - .fields() - .iter() - .find(|f| f.names().contains(&field.name().to_string())); - let mut metadata = field.metadata().clone(); - if let Some(mapped_field) = mapped_field_opt - && let Some(field_id) = mapped_field.field_id() + if let Some(field_id) = field_ids_by_name + .get(field.name().as_str()) + .copied() + .flatten() { // Field found in mapping with a field_id → assign it metadata.insert(PARQUET_FIELD_ID_META_KEY.to_string(), field_id.to_string()); @@ -523,6 +530,34 @@ mod tests { }; use crate::{ErrorKind, Runtime}; + #[test] + fn test_name_mapping_duplicate_alias_preserves_first_match() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "name", + DataType::Utf8, + true, + )])); + for (first_id, expected) in [(Some(2), Some("2")), (None, None)] { + let name_mapping = NameMapping::new(vec![ + MappedField::new(first_id, vec!["name".to_string()], vec![]), + MappedField::new(Some(4), vec!["name".to_string()], vec![]), + ]); + + let mapped = + super::apply_name_mapping_to_arrow_schema(arrow_schema.clone(), &name_mapping) + .unwrap(); + + assert_eq!( + mapped.fields()[0] + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .map(String::as_str), + expected, + "first matching field ID: {first_id:?}" + ); + } + } + #[test] fn test_arrow_projection_mask() { let schema = Arc::new( diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 5d92f6bc7b..4a99914a34 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -35,6 +35,7 @@ use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; use crate::spec::Schema; +use crate::{Error, ErrorKind}; impl ArrowReader { pub(super) fn get_row_filter( @@ -71,11 +72,21 @@ impl ArrowReader { parquet_metadata: &Arc, field_id_map: &HashMap, snapshot_schema: &Schema, + candidate_row_groups: &[usize], ) -> Result> { let row_groups_metadata = parquet_metadata.row_groups(); - let mut results = Vec::with_capacity(row_groups_metadata.len()); - - for (idx, row_group_metadata) in row_groups_metadata.iter().enumerate() { + let mut results = Vec::with_capacity(candidate_row_groups.len()); + + for &idx in candidate_row_groups { + let row_group_metadata = row_groups_metadata.get(idx).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "row group index {idx} out of range ({} row groups)", + row_groups_metadata.len() + ), + ) + })?; if RowGroupMetricsEvaluator::eval( predicate, row_group_metadata, @@ -634,6 +645,67 @@ mod tests { assert_eq!(first_val, 100, "Task 2 should start with id=100, not id=0"); } + + // Combining byte-range and predicate row-group filtering should narrow the + // split's candidates instead of evaluating and intersecting independent lists. + // This task owns row groups 1 and 2, while the predicate can match row groups + // 0 and 2. Only row group 2 should be read. + let predicate = Reference::new("id") + .less_than(Datum::int(100)) + .or(Reference::new("id").greater_than_or_equal_to(Datum::int(200))) + .bind(schema.clone(), true) + .unwrap(); + + // Test the pruning decision directly, not just the final filtered rows. + let parquet_metadata = Arc::new((*metadata).clone()); + let field_id_map = HashMap::from([(1, 0)]); + for (candidates, expected) in [(vec![1, 2], vec![2]), (vec![0, 1], vec![0])] { + let selected = ArrowReader::get_selected_row_group_indices( + &predicate, + &parquet_metadata, + &field_id_map, + schema.as_ref(), + &candidates, + ) + .unwrap(); + assert_eq!(selected, expected); + } + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(rg1_start) + .with_length(file_end - rg1_start) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_predicate(Some(predicate)) + .with_record_count(Some(200)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_row_group_filtering_enabled(true) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let mut ids = Vec::new(); + for batch in &result { + let id_col = batch + .column(0) + .as_primitive::(); + ids.extend((0..id_col.len()).map(|index| id_col.value(index))); + } + + assert_eq!(ids, (200..300).collect::>()); } /// A single data file split into multiple sub-row-group byte ranges (as Spark/Iceberg @@ -903,6 +975,24 @@ mod tests { ) } + #[test] + fn test_row_group_pruning_rejects_invalid_candidate() { + let schema = int_schema(); + let predicate = simple_predicate(schema.clone()); + let metadata = metadata_no_page_indexes(); + + let err = ArrowReader::get_selected_row_group_indices( + &predicate, + &metadata, + &field_id_map(), + schema.as_ref(), + &[0], + ) + .unwrap_err(); + + assert_eq!(err.kind(), crate::ErrorKind::DataInvalid); + } + /// Testing suite regarding: https://github.com/apache/iceberg-rust/issues/2452 /// Testing when: both indices are absent, some present, both present #[test] From 693339072b87eebf075479393a8ef55182f27aca Mon Sep 17 00:00:00 2001 From: DeviousCardi <115358213+DeviousCardi@users.noreply.github.com> Date: Mon, 12 Oct 2026 08:48:39 +0000 Subject: [PATCH 59/63] fix(arrow): decode short BYTE_ARRAY decimal row-group stats (#3364) get_parquet_stat_min_as_datum / get_parquet_stat_max_as_datum decoded BYTE_ARRAY decimal bounds with a strict 16-byte conversion. Parquet stores these as minimal-length big-endian two's-complement values, so any bound shorter than 16 bytes failed to convert and aborted the row-group read. Use the variable-length, sign-extending i128_from_be_bytes helper, as the FIXED_LEN_BYTE_ARRAY arm already does. Bounds longer than 16 bytes still return DataInvalid. Empty bounds also return DataInvalid instead of decoding as zero, which could otherwise prune matching row groups. Closes #3280 --- crates/iceberg/src/arrow/schema.rs | 95 +++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 996ced09ef..0c673c6bc5 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -893,7 +893,13 @@ pub(crate) fn get_parquet_stat_min_as_datum( }; Some(Datum::new( primitive_type.clone(), - PrimitiveLiteral::Int128(i128::from_be_bytes(bytes.try_into()?)), + PrimitiveLiteral::Int128( + // An empty bound carries no value; reject it rather than + // decode it as zero, which could prune matching row groups. + i128_from_be_bytes(bytes) + .filter(|_| !bytes.is_empty()) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), )) } ( @@ -1037,7 +1043,13 @@ pub(crate) fn get_parquet_stat_max_as_datum( }; Some(Datum::new( primitive_type.clone(), - PrimitiveLiteral::Int128(i128::from_be_bytes(bytes.try_into()?)), + PrimitiveLiteral::Int128( + // An empty bound carries no value; reject it rather than + // decode it as zero, which could prune matching row groups. + i128_from_be_bytes(bytes) + .filter(|_| !bytes.is_empty()) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), )) } ( @@ -2625,6 +2637,85 @@ mod tests { assert_eq!(schema.highest_field_id(), 17); } + fn decimal_byte_array_stats(min: Vec, max: Vec) -> Statistics { + Statistics::byte_array( + Some(parquet::data_type::ByteArray::from(min)), + Some(parquet::data_type::ByteArray::from(max)), + None, + Some(0), + false, + ) + } + + fn decimal_bounds(stats: &Statistics) -> Result<(Option, Option)> { + let ty = PrimitiveType::Decimal { + precision: 38, + scale: 2, + }; + Ok(( + get_parquet_stat_min_as_datum(&ty, stats)?, + get_parquet_stat_max_as_datum(&ty, stats)?, + )) + } + + fn decimal_datum(value: i128) -> Datum { + Datum::new( + PrimitiveType::Decimal { + precision: 38, + scale: 2, + }, + PrimitiveLiteral::Int128(value), + ) + } + + #[test] + fn test_byte_array_decimal_stats_short_bounds() { + // Minimal-length big-endian two's-complement: -1234 and 1234. + let stats = decimal_byte_array_stats(vec![0xFB, 0x2E], vec![0x04, 0xD2]); + let (min, max) = decimal_bounds(&stats).unwrap(); + assert_eq!(min, Some(decimal_datum(-1234))); + assert_eq!(max, Some(decimal_datum(1234))); + + // Single-byte bounds: sign bit set must sign-extend, not zero-extend. + let stats = decimal_byte_array_stats(vec![0x80], vec![0x7F]); + let (min, max) = decimal_bounds(&stats).unwrap(); + assert_eq!(min, Some(decimal_datum(-128))); + assert_eq!(max, Some(decimal_datum(127))); + } + + #[test] + fn test_byte_array_decimal_stats_full_width_bounds() { + let stats = decimal_byte_array_stats( + i128::MIN.to_be_bytes().to_vec(), + i128::MAX.to_be_bytes().to_vec(), + ); + let (min, max) = decimal_bounds(&stats).unwrap(); + assert_eq!(min, Some(decimal_datum(i128::MIN))); + assert_eq!(max, Some(decimal_datum(i128::MAX))); + } + + #[test] + fn test_byte_array_decimal_stats_empty_bound_errors() { + let stats = decimal_byte_array_stats(vec![], vec![0x01]); + let err = decimal_bounds(&stats).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + + let stats = decimal_byte_array_stats(vec![0x01], vec![]); + let err = decimal_bounds(&stats).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_byte_array_decimal_stats_oversized_bound_errors() { + let stats = decimal_byte_array_stats(vec![0; 17], vec![0x01]); + let err = decimal_bounds(&stats).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + + let stats = decimal_byte_array_stats(vec![0x01], vec![0; 17]); + let err = decimal_bounds(&stats).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + /// Every field is rebuilt without metadata, at any depth and inside any /// container. Fields are rebuilt in nesting order rather than looked up by /// name, so a path that repeats names (`a.b.a.b`) keeps each level's name From b55cda1cb9e503336bdd41e2b2489415216577d7 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Mon, 12 Oct 2026 18:27:09 +0000 Subject: [PATCH 60/63] fix(reader): apply name mapping to nested fields (#3384) * fix(reader): apply name mapping to nested fields apply_name_mapping_to_arrow_schema assigned field IDs to top-level columns only. The projection then never selected the leaves of a struct, list, or map in a file without embedded IDs, and those columns read as NULL whenever a top-level column was projected too. Apply the mapping at every depth, as Java's ApplyNameMapping does. Struct children are looked up within their parent's entry, and list elements and map keys and values as element, key, and value, whatever the file calls them. Closes #3374 * Size collections up front, shorten comments, dedupe tests - Build each rebuilt Fields from Vec::with_capacity instead of a fallible collect, and the map entries from a two-element array. - Size the maps and index list in get_arrow_projection_mask_with_field_ids. - Keep one reader test for the reported struct case and cover every nested Arrow type in one schema-level test. * Address review: skip entries without a field ID, pin types and errors - Treat a mapping entry without a field ID as missing, so its nested fields keep no IDs either. - Note that the map shape checks are defensive, and report the entry count when one fails. - Note that the projection treats ListView and LargeListView as leaves. - Test unmapped children, an entry without a field ID, the fixed-size list size, the map ordering flag, and a predicate on a nested field. --- crates/iceberg/src/arrow/reader/projection.rs | 357 ++++++++++++++++-- 1 file changed, 322 insertions(+), 35 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index 2fa6bf18f9..b76d60c856 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -23,7 +23,9 @@ use std::collections::{HashMap, HashSet}; use std::str::FromStr; use std::sync::Arc; -use arrow_schema::{Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef}; +use arrow_schema::{ + DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, +}; use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ProjectionMask}; use parquet::schema::types::{SchemaDescriptor, Type as ParquetType}; @@ -32,7 +34,10 @@ use crate::arrow::arrow_schema_to_schema; use crate::error::{Result, invalid_data}; use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; -use crate::spec::{NameMapping, NestedField, PrimitiveType, Schema, Type}; +use crate::spec::{ + LIST_FIELD_NAME, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MappedField, NameMapping, + NestedField, PrimitiveType, Schema, Type, +}; use crate::{Error, ErrorKind}; impl ArrowReader { @@ -171,7 +176,7 @@ impl ArrowReader { arrow_schema: &ArrowSchemaRef, type_promotion_is_valid: fn(Option<&PrimitiveType>, Option<&PrimitiveType>) -> bool, ) -> Result { - let mut column_map = HashMap::new(); + let mut column_map = HashMap::with_capacity(leaf_field_ids.len()); let fields = arrow_schema.fields(); // HashSet for O(1) membership checks instead of O(n) slice scans. let leaf_field_id_set: HashSet = leaf_field_ids.iter().copied().collect(); @@ -216,7 +221,7 @@ impl ArrowReader { // We only project existing columns; RecordBatchTransformer adds default/NULL values. // If other leaves are selected, a struct whose requested leaves are all missing gets no // column and reads as NULL (#3378). - let mut indices = vec![]; + let mut indices = Vec::with_capacity(leaf_field_ids.len()); for field_id in leaf_field_ids { if let Some(col_idx) = column_map.get(field_id) { indices.push(*col_idx); @@ -392,7 +397,7 @@ fn parquet_field_id(field: &Field) -> Option { /// Apply name mapping to Arrow schema for Parquet files lacking field IDs. /// -/// Assigns Iceberg field IDs based on column names using the name mapping, +/// Assigns Iceberg field IDs based on names at every depth using the name mapping, /// enabling correct projection on migrated files (e.g., from Hive/Spark via add_files). /// /// Per Iceberg spec Column Projection rule #2: @@ -421,43 +426,20 @@ pub(super) fn apply_name_mapping_to_arrow_schema( "Schema already has field IDs - name mapping should not be applied" ); - let mut field_ids_by_name = HashMap::with_capacity(name_mapping.fields().len()); + let mut mapped_fields_by_name = HashMap::with_capacity(name_mapping.fields().len()); for mapped_field in name_mapping.fields() { for name in mapped_field.names() { // Preserve the existing first-match behavior for duplicate aliases. - field_ids_by_name + mapped_fields_by_name .entry(name.as_str()) - .or_insert(mapped_field.field_id()); + .or_insert(mapped_field); } } - let fields_with_mapped_ids: Vec<_> = arrow_schema - .fields() - .iter() - .map(|field| { - // Look up this column name in name mapping to get the Iceberg field ID. - // Corresponds to Java's ApplyNameMapping visitor which calls - // nameMapping.find(currentPath()) and returns field.withId() if found. - // - // If the field isn't in the mapping, leave it WITHOUT assigning an ID - // (matching Java's behavior of returning the field unchanged). - // Later, during projection, fields without IDs are filtered out. - let mut metadata = field.metadata().clone(); - - if let Some(field_id) = field_ids_by_name - .get(field.name().as_str()) - .copied() - .flatten() - { - // Field found in mapping with a field_id → assign it - metadata.insert(PARQUET_FIELD_ID_META_KEY.to_string(), field_id.to_string()); - } - // If field_id is None, leave the field without an ID (will be filtered by projection) - - Field::new(field.name(), field.data_type().clone(), field.is_nullable()) - .with_metadata(metadata) - }) - .collect(); + let fields_with_mapped_ids = try_map_fields(arrow_schema.fields(), |field| { + let mapped_field = mapped_fields_by_name.get(field.name().as_str()).copied(); + apply_name_mapping_to_field(field, mapped_field) + })?; Ok(Arc::new(ArrowSchema::new_with_metadata( fields_with_mapped_ids, @@ -465,6 +447,92 @@ pub(super) fn apply_name_mapping_to_arrow_schema( ))) } +/// Assigns IDs from `mapped_field` to `field` and its nested fields, as Java's ApplyNameMapping +/// does. List elements match `element`, and map entries match `key` and `value`. A field +/// without an entry, or whose entry has no field ID, keeps no ID, and so do its nested fields. +fn apply_name_mapping_to_field( + field: &FieldRef, + mapped_field: Option<&MappedField>, +) -> Result { + let Some((mapped_field, field_id)) = + mapped_field.zip(mapped_field.and_then(MappedField::field_id)) + else { + return Ok(Arc::clone(field)); + }; + let map_nested = |nested: &FieldRef, name: &str| { + let mapped_nested = find_mapped_field(mapped_field.fields().iter().map(Arc::as_ref), name); + apply_name_mapping_to_field(nested, mapped_nested) + }; + + let data_type = match field.data_type() { + DataType::Struct(fields) => DataType::Struct(try_map_fields(fields, |nested| { + map_nested(nested, nested.name()) + })?), + DataType::List(element) => DataType::List(map_nested(element, LIST_FIELD_NAME)?), + DataType::LargeList(element) => DataType::LargeList(map_nested(element, LIST_FIELD_NAME)?), + DataType::FixedSizeList(element, size) => { + DataType::FixedSizeList(map_nested(element, LIST_FIELD_NAME)?, *size) + } + // The mapping has no entry for Arrow's entries struct. The shape checks are defensive, + // because parquet-rs builds every map as a struct of key and value. + DataType::Map(entries, sorted) => { + let DataType::Struct(entry_fields) = entries.data_type() else { + return Err(invalid_data!( + "Map field `{}` must have struct type", + field.name() + )); + }; + let [key, value] = &entry_fields[..] else { + return Err(invalid_data!( + "Map field `{}` must have exactly 2 fields, found {}", + field.name(), + entry_fields.len() + )); + }; + let entry_fields = Fields::from([ + map_nested(key, MAP_KEY_FIELD_NAME)?, + map_nested(value, MAP_VALUE_FIELD_NAME)?, + ]); + let entries = entries + .as_ref() + .clone() + .with_data_type(DataType::Struct(entry_fields)); + DataType::Map(Arc::new(entries), *sorted) + } + // `Fields::filter_leaves` treats `ListView` and `LargeListView` as leaves, so projection + // cannot select their elements even with IDs. + other => other.clone(), + }; + + let mut field = field.as_ref().clone().with_data_type(data_type); + field + .metadata_mut() + .insert(PARQUET_FIELD_ID_META_KEY.to_string(), field_id.to_string()); + Ok(Arc::new(field)) +} + +/// Returns the first entry in `mapped_fields` that lists `name`. +fn find_mapped_field<'a>( + mapped_fields: impl IntoIterator, + name: &str, +) -> Option<&'a MappedField> { + mapped_fields + .into_iter() + .find(|mapped_field| mapped_field.names().iter().any(|n| n == name)) +} + +/// Maps `fields` into a `Fields` sized up front, which `collect::>()` does not do. +fn try_map_fields( + fields: &Fields, + mut map_field: impl FnMut(&FieldRef) -> Result, +) -> Result { + let mut mapped = Vec::with_capacity(fields.len()); + for field in fields { + mapped.push(map_field(field)?); + } + Ok(Fields::from(mapped)) +} + /// Add position-based fallback field IDs to Arrow schema for Parquet files lacking them. /// Enables projection on migrated files (e.g., from Hive/Spark). /// @@ -1106,6 +1174,225 @@ message schema { assert_eq!(subdept_array.value(0), "Bob"); } + /// Regression test for #3374: a struct in a name-mapped file reads correctly next to a + /// top-level column, and predicates resolve its fields. + #[tokio::test] + async fn test_read_parquet_with_name_mapping_maps_nested_fields() { + use arrow_array::types::Int32Type; + use arrow_array::{Int32Array, StructArray}; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "a", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let name_mapping = Arc::new(NameMapping::new(vec![ + MappedField::new(Some(1), vec!["id".to_string()], vec![]), + MappedField::new(Some(2), vec!["s".to_string()], vec![MappedField::new( + Some(3), + vec!["a".to_string()], + vec![], + )]), + ])); + + // No field IDs at any level. + let s_col = StructArray::from(vec![( + Arc::new(Field::new("a", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef, + )]); + let to_write = RecordBatch::try_from_iter_with_nullable(vec![ + ( + "id", + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + true, + ), + ("s", Arc::new(s_col) as ArrayRef, true), + ]) + .unwrap(); + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/1.parquet", tmp_dir.path().to_str().unwrap()); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), None).unwrap(); + writer.write(&to_write).unwrap(); + writer.close().unwrap(); + + let file_size = std::fs::metadata(&file_path).unwrap().len(); + let tasks = |predicate| { + Box::pin(futures::stream::iter([Ok(FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .with_name_mapping(Some(name_mapping.clone())) + .with_predicate(predicate) + .build() + .unwrap())])) as FileScanTaskStream + }; + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + + let result = reader + .clone() + .read(tasks(None)) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + let s = result[0].column(1).as_struct(); + assert_eq!(s.null_count(), 0); + assert_eq!( + s.column(0).as_primitive::(), + &Int32Array::from(vec![10, 20, 30]) + ); + + // `s.a` resolves, so the row filter rejects the nested predicate as it does for files + // with embedded IDs, rather than matching no rows. + let predicate = Reference::new("s.a") + .greater_than(Datum::int(15)) + .bind(schema.clone(), true) + .unwrap(); + let err = reader + .read(tasks(Some(predicate))) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + /// Every nested Arrow type gets IDs, with list and map entries matched by their spec names. + #[test] + fn test_apply_name_mapping_to_nested_arrow_types() { + use arrow_schema::Fields; + + use super::apply_name_mapping_to_arrow_schema; + use crate::arrow::arrow_schema_to_schema; + + let field = |name: &str, data_type| Field::new(name, data_type, true); + let item = |data_type| Arc::new(field("item", data_type)); + let entries = Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + field("values", DataType::Int32), + ])), + false, + ); + let x = Fields::from(vec![field("x", DataType::Int32)]); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + field("id", DataType::Int32), + // `s.id` must take its ID from the entry for `s`, not from the top-level `id`. + field( + "s", + DataType::Struct(Fields::from(vec![field("id", DataType::Int32)])), + ), + field("list", DataType::List(item(DataType::Int32))), + field("fixed", DataType::FixedSizeList(item(DataType::Int32), 2)), + field("large", DataType::LargeList(item(DataType::Struct(x)))), + field("map", DataType::Map(Arc::new(entries), true)), + ])); + let entry = + |id, name: &str, fields| MappedField::new(Some(id), vec![name.to_string()], fields); + let name_mapping = NameMapping::new(vec![ + entry(1, "id", vec![]), + entry(2, "s", vec![entry(3, "id", vec![])]), + entry(4, "list", vec![entry(5, "element", vec![])]), + entry(6, "fixed", vec![entry(7, "element", vec![])]), + entry(8, "large", vec![entry(9, "element", vec![entry( + 10, + "x", + vec![], + )])]), + entry(11, "map", vec![ + entry(12, "key", vec![]), + entry(13, "value", vec![]), + ]), + ]); + + let mapped = apply_name_mapping_to_arrow_schema(arrow_schema, &name_mapping).unwrap(); + + // The list size and the map ordering carry over. + assert!(matches!( + mapped.field(3).data_type(), + DataType::FixedSizeList(_, 2) + )); + assert!(matches!( + mapped.field(5).data_type(), + DataType::Map(_, true) + )); + // The conversion fails on any field without an ID. + let schema = arrow_schema_to_schema(&mapped).unwrap(); + let names: Vec<_> = (1..=13) + .map(|id| schema.name_by_field_id(id).unwrap_or_default()) + .collect(); + assert_eq!(names, [ + "id", + "s", + "s.id", + "list", + "list.element", + "fixed", + "fixed.element", + "large", + "large.element", + "large.element.x", + "map", + "map.key", + "map.value", + ]); + } + + /// A field without an entry, or whose entry has no field ID, keeps no ID, and so do its + /// nested fields. + #[test] + fn test_apply_name_mapping_skips_unmapped_fields() { + use arrow_schema::Fields; + + use super::{apply_name_mapping_to_arrow_schema, build_field_id_map_from_arrow_schema}; + + let int = |name: &str| Field::new(name, DataType::Int32, true); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "s", + DataType::Struct(Fields::from(vec![int("a"), int("b")])), + true, + ), + Field::new("t", DataType::Struct(Fields::from(vec![int("c")])), true), + ])); + let entry = |id, name: &str, fields| MappedField::new(id, vec![name.to_string()], fields); + let name_mapping = NameMapping::new(vec![ + // No entry for `s.b`. + entry(Some(1), "s", vec![entry(Some(2), "a", vec![])]), + entry(None, "t", vec![entry(Some(3), "c", vec![])]), + ]); + + let mapped = apply_name_mapping_to_arrow_schema(arrow_schema, &name_mapping).unwrap(); + + // Only `s.a`, leaf 0, resolves to an ID. + assert_eq!( + build_field_id_map_from_arrow_schema(&mapped), + HashMap::from([(2, 0)]) + ); + } + /// Test reading Parquet files without field IDs with partial projection. /// Only a subset of columns are requested, verifying position-based fallback /// handles column selection correctly. From b154603a710d730d2f43f8af4db257cc137237ef Mon Sep 17 00:00:00 2001 From: advancedxy <807537+advancedxy@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:41:02 +0000 Subject: [PATCH 61/63] fix(transform): correct decimal and binary truncate literals (#3243) * fix(transform): correct decimal and binary truncate literals The Arrow array path already preserves decimal precision and scale and supports binary truncation. The literal path instead reset decimal scale to zero and rejected binary values. For example, truncate[50](10.65) returned 1050 instead of 10.50. Align literal truncation with the Arrow array path: preserve the source decimal precision and scale, and return the first W bytes of binary values, or the entire value when shorter. Decimal projection boundary adjustments must also preserve precision and scale to keep their values correct before truncation. * fix(transform): address truncate literal review feedback --- .../src/expr/visitors/strict_projection.rs | 24 +- crates/iceberg/src/spec/transform.rs | 13 +- crates/iceberg/src/spec/values/datum.rs | 16 ++ crates/iceberg/src/transform/truncate.rs | 241 ++++++++++++++++-- 4 files changed, 251 insertions(+), 43 deletions(-) diff --git a/crates/iceberg/src/expr/visitors/strict_projection.rs b/crates/iceberg/src/expr/visitors/strict_projection.rs index ebc6212c76..2188e6df08 100644 --- a/crates/iceberg/src/expr/visitors/strict_projection.rs +++ b/crates/iceberg/src/expr/visitors/strict_projection.rs @@ -2761,7 +2761,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 10000)".to_string() + "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 100.00)".to_string() ); // test less or eq @@ -2780,7 +2780,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 10000)".to_string() + "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 100.00)".to_string() ); // test greater @@ -2799,7 +2799,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 > 100) AND (pcol2 > 100)) AND (pcol3 > 10000)".to_string() + "((pcol1 > 100) AND (pcol2 > 100)) AND (pcol3 > 100.00)".to_string() ); // test greater or eq @@ -2818,7 +2818,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 9990)".to_string() + "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 99.90)".to_string() ); // test not eq @@ -2837,7 +2837,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 != 100) AND (pcol2 != 100)) AND (pcol3 != 10000)".to_string() + "((pcol1 != 100) AND (pcol2 != 100)) AND (pcol3 != 100.00)".to_string() ); // test not in @@ -2880,7 +2880,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 NOT IN (100, 90)) AND (pcol2 NOT IN (100, 90))) AND (pcol3 NOT IN (10000, 10100, 9900))" + "((pcol1 NOT IN (100, 90)) AND (pcol2 NOT IN (100, 90))) AND (pcol3 NOT IN (100.00, 99.00, 101.00))" .to_string() ); @@ -2984,7 +2984,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 < 90) AND (pcol2 < 90)) AND (pcol3 < 9990)".to_string() + "((pcol1 < 90) AND (pcol2 < 90)) AND (pcol3 < 99.90)".to_string() ); // test less or eq @@ -3003,7 +3003,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 10000)".to_string() + "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 100.00)".to_string() ); // test greater @@ -3022,7 +3022,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 9990)".to_string() + "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 99.90)".to_string() ); // test greater or eq @@ -3041,7 +3041,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 9990)".to_string() + "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 99.90)".to_string() ); // test not eq @@ -3060,7 +3060,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 != 90) AND (pcol2 != 90)) AND (pcol3 != 9990)".to_string() + "((pcol1 != 90) AND (pcol2 != 90)) AND (pcol3 != 99.90)".to_string() ); // test not in @@ -3103,7 +3103,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 NOT IN (100, 90)) AND (pcol2 NOT IN (100, 90))) AND (pcol3 NOT IN (9890, 9990, 10090))" + "((pcol1 NOT IN (100, 90)) AND (pcol2 NOT IN (100, 90))) AND (pcol3 NOT IN (98.90, 100.90, 99.90))" .to_string() ); diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index d87054ab73..c28dc28c19 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -25,7 +25,6 @@ use chrono::Datelike; use fnv::FnvHashSet; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use super::values::decimal_utils::decimal_from_i128_with_scale; use super::values::temporal::date; use super::{Datum, PrimitiveLiteral}; use crate::ErrorKind; @@ -762,8 +761,8 @@ impl Transform { PredicateOperator::LessThan => match (datum.data_type(), datum.literal()) { (PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Some(Datum::int(v - 1)), (PrimitiveType::Long, PrimitiveLiteral::Long(v)) => Some(Datum::long(v - 1)), - (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => { - Some(Datum::decimal(decimal_from_i128_with_scale(v - 1, 0))?) + (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(_)) => { + Some(Self::try_decrement_number(datum)?) } (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => Some(Datum::date(v - 1)), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { @@ -783,8 +782,8 @@ impl Transform { PredicateOperator::GreaterThan => match (datum.data_type(), datum.literal()) { (PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Some(Datum::int(v + 1)), (PrimitiveType::Long, PrimitiveLiteral::Long(v)) => Some(Datum::long(v + 1)), - (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => { - Some(Datum::decimal(decimal_from_i128_with_scale(v + 1, 0))?) + (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(_)) => { + Some(Self::try_increment_number(datum)?) } (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => Some(Datum::date(v + 1)), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { @@ -934,7 +933,7 @@ impl Transform { (PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Ok(Datum::int(v + 1)), (PrimitiveType::Long, PrimitiveLiteral::Long(v)) => Ok(Datum::long(v + 1)), (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => { - Datum::decimal(decimal_from_i128_with_scale(v + 1, 0)) + datum.with_decimal_mantissa(v + 1) } (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => Ok(Datum::date(v + 1)), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { @@ -972,7 +971,7 @@ impl Transform { (PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Ok(Datum::int(v - 1)), (PrimitiveType::Long, PrimitiveLiteral::Long(v)) => Ok(Datum::long(v - 1)), (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => { - Datum::decimal(decimal_from_i128_with_scale(v - 1, 0)) + datum.with_decimal_mantissa(v - 1) } (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => Ok(Datum::date(v - 1)), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index 57d9675736..c4c835f61f 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -1099,6 +1099,22 @@ impl Datum { } } + /// Replaces a decimal's unscaled value while preserving its precision and scale. + /// + /// The value is not checked against precision because transform projections + /// may use boundaries outside the source type's range. + pub(crate) fn with_decimal_mantissa(&self, mantissa: i128) -> Result { + ensure_data_valid!( + matches!(self.r#type, PrimitiveType::Decimal { .. }), + "Expected a decimal datum, got: {:?}", + self.r#type + ); + Ok(Self::new( + self.r#type.clone(), + PrimitiveLiteral::Int128(mantissa), + )) + } + /// Convert the datum to `target_type`. pub fn to(self, target_type: &Type) -> Result { match target_type { diff --git a/crates/iceberg/src/transform/truncate.rs b/crates/iceberg/src/transform/truncate.rs index 0f95854ee1..2bb7fdf22b 100644 --- a/crates/iceberg/src/transform/truncate.rs +++ b/crates/iceberg/src/transform/truncate.rs @@ -23,8 +23,7 @@ use arrow_schema::DataType; use super::TransformFunction; use crate::Error; use crate::error::invalid_data; -use crate::spec::decimal_utils::decimal_from_i128_with_scale; -use crate::spec::{Datum, PrimitiveLiteral}; +use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType}; #[derive(Debug)] pub struct Truncate { @@ -159,15 +158,20 @@ impl TransformFunction for Truncate { })), PrimitiveLiteral::Int128(v) => Ok(Some({ let width = self.width as i128; - Datum::decimal(decimal_from_i128_with_scale( - Self::truncate_decimal_i128(*v, width), - 0, - ))? + input.with_decimal_mantissa(Self::truncate_decimal_i128(*v, width))? })), PrimitiveLiteral::String(v) => Ok(Some({ let len = self.width as usize; Datum::string(Self::truncate_str(v, len).to_string()) })), + // Fixed shares this literal variant with Binary but does not support truncate. + PrimitiveLiteral::Binary(v) if matches!(input.data_type(), PrimitiveType::Binary) => { + Ok(Some(Datum::binary( + Self::truncate_binary(v, self.width as usize) + .iter() + .copied(), + ))) + } _ => Err(Error::new( crate::ErrorKind::FeatureUnsupported, format!( @@ -194,8 +198,7 @@ mod test { TimestampNs, Timestamptz, TimestamptzNs, Uuid, }; use crate::spec::Type::{Primitive, Struct}; - use crate::spec::decimal_utils::decimal_new; - use crate::spec::{Datum, NestedField, PrimitiveType, StructType, Transform}; + use crate::spec::{Datum, NestedField, PrimitiveLiteral, PrimitiveType, StructType, Transform}; use crate::transform::TransformFunction; use crate::transform::test::{TestProjectionFixture, TestTransformFixture}; @@ -250,6 +253,128 @@ mod test { fixture.assert_transform(trans); } + #[test] + fn test_projection_truncate_binary() -> Result<()> { + let fixture = TestProjectionFixture::new( + Transform::Truncate(3), + "name", + NestedField::required(1, "value", Primitive(Binary)), + ); + + for (value, prefix) in [ + (vec![0x00, 0xff, 0x80, 0x01], "00FF80"), + (vec![0x00, 0xff, 0x80], "00FF80"), + (vec![0x00, 0xff], "00FF"), + ] { + for (op, projected_op) in [ + (PredicateOperator::Eq, "="), + (PredicateOperator::LessThanOrEq, "<="), + ] { + fixture.assert_projection( + &fixture.binary_predicate(op, Datum::binary(value.iter().copied())), + Some(&format!("name {projected_op} {prefix}")), + )?; + } + } + + // Both source values share one truncated prefix, so the projected set has one value. + fixture.assert_projection( + &fixture.set_predicate(PredicateOperator::In, vec![ + Datum::binary([0x00, 0xff, 0x80, 0x01]), + Datum::binary([0x00, 0xff, 0x80, 0x02]), + ]), + Some("name IN (00FF80)"), + )?; + + Ok(()) + } + + #[test] + fn test_strict_projection_truncate_binary() -> Result<()> { + let transform = Transform::Truncate(3); + let fixture = TestProjectionFixture::new( + transform, + "name", + NestedField::required(1, "value", Primitive(Binary)), + ); + + for (op, expected) in [ + (PredicateOperator::LessThanOrEq, Some("name < 00FF80")), + (PredicateOperator::NotEq, Some("name != 00FF80")), + (PredicateOperator::Eq, None), + ] { + let predicate = fixture.binary_predicate(op, Datum::binary([0x00, 0xff, 0x80, 0x01])); + let result = transform.strict_project("name", &predicate)?; + assert_eq!(result.map(|p| p.to_string()).as_deref(), expected); + } + + for (op, expected) in [ + (PredicateOperator::NotIn, Some("name NOT IN (00FF80)")), + (PredicateOperator::In, None), + ] { + let predicate = fixture.set_predicate(op, vec![ + Datum::binary([0x00, 0xff, 0x80, 0x01]), + Datum::binary([0x00, 0xff, 0x80, 0x02]), + ]); + let result = transform.strict_project("name", &predicate)?; + assert_eq!(result.map(|p| p.to_string()).as_deref(), expected); + } + + Ok(()) + } + + #[test] + fn test_projection_truncate_decimal_outside_precision() -> Result<()> { + let transform = Transform::Truncate(10); + let fixture = TestProjectionFixture::new( + transform, + "name", + NestedField::required( + 1, + "value", + Primitive(Decimal { + precision: 2, + scale: 1, + }), + ), + ); + + for (value, inclusive_op, inclusive_expected, strict_op, strict_expected) in [ + ( + 99, + PredicateOperator::GreaterThan, + "name >= 10.0", + PredicateOperator::LessThanOrEq, + "name < 10.0", + ), + ( + -99, + PredicateOperator::LessThan, + "name <= -10.0", + PredicateOperator::GreaterThanOrEq, + "name > -10.0", + ), + ] { + let datum = Datum::new( + Decimal { + precision: 2, + scale: 1, + }, + PrimitiveLiteral::Int128(value), + ); + fixture.assert_projection( + &fixture.binary_predicate(inclusive_op, datum.clone()), + Some(inclusive_expected), + )?; + let result = transform + .strict_project("name", &fixture.binary_predicate(strict_op, datum))? + .unwrap(); + assert_eq!(result.to_string(), strict_expected); + } + + Ok(()) + } + #[test] fn test_projection_truncate_string_rewrite_op() -> Result<()> { let value = "abcde"; @@ -359,7 +484,7 @@ mod test { fixture.assert_projection( &fixture.binary_predicate(PredicateOperator::LessThan, Datum::decimal_from_str(curr)?), - Some("name <= 9990"), + Some("name <= 99.90"), )?; fixture.assert_projection( @@ -367,7 +492,7 @@ mod test { PredicateOperator::LessThanOrEq, Datum::decimal_from_str(curr)?, ), - Some("name <= 9990"), + Some("name <= 99.90"), )?; fixture.assert_projection( @@ -375,12 +500,12 @@ mod test { PredicateOperator::GreaterThanOrEq, Datum::decimal_from_str(curr)?, ), - Some("name >= 9990"), + Some("name >= 99.90"), )?; fixture.assert_projection( &fixture.binary_predicate(PredicateOperator::Eq, Datum::decimal_from_str(curr)?), - Some("name = 9990"), + Some("name = 99.90"), )?; fixture.assert_projection( @@ -394,7 +519,7 @@ mod test { Datum::decimal_from_str(curr)?, Datum::decimal_from_str(next)?, ]), - Some("name IN (9890, 9990, 10090)"), + Some("name IN (99.90, 100.90, 98.90)"), )?; fixture.assert_projection( @@ -429,7 +554,7 @@ mod test { fixture.assert_projection( &fixture.binary_predicate(PredicateOperator::LessThan, Datum::decimal_from_str(curr)?), - Some("name <= 9990"), + Some("name <= 99.90"), )?; fixture.assert_projection( @@ -437,7 +562,7 @@ mod test { PredicateOperator::LessThanOrEq, Datum::decimal_from_str(curr)?, ), - Some("name <= 10000"), + Some("name <= 100.00"), )?; fixture.assert_projection( @@ -445,12 +570,12 @@ mod test { PredicateOperator::GreaterThanOrEq, Datum::decimal_from_str(curr)?, ), - Some("name >= 10000"), + Some("name >= 100.00"), )?; fixture.assert_projection( &fixture.binary_predicate(PredicateOperator::Eq, Datum::decimal_from_str(curr)?), - Some("name = 10000"), + Some("name = 100.00"), )?; fixture.assert_projection( @@ -464,7 +589,7 @@ mod test { Datum::decimal_from_str(curr)?, Datum::decimal_from_str(next)?, ]), - Some("name IN (10000, 10100, 9900)"), + Some("name IN (99.00, 100.00, 101.00)"), )?; fixture.assert_projection( @@ -795,6 +920,34 @@ mod test { assert_eq!(super::Truncate::truncate_str(test4, 1), test4_1_expected); } + #[test] + fn test_literal_spec_examples() -> Result<()> { + // https://iceberg.apache.org/spec/#truncate-transform-details + let cases = [ + (10, Datum::int(1), Datum::int(0)), + (10, Datum::int(-1), Datum::int(-10)), + (10, Datum::long(1), Datum::long(0)), + (10, Datum::long(-1), Datum::long(-10)), + ( + 50, + Datum::decimal_from_str("10.65")?, + Datum::decimal_from_str("10.50")?, + ), + (3, Datum::string("iceberg"), Datum::string("ice")), + ( + 3, + Datum::binary(vec![0x01, 0x02, 0x03, 0x04, 0x05]), + Datum::binary(vec![0x01, 0x02, 0x03]), + ), + ]; + + for (width, input, expected) in cases { + let result = super::Truncate::new(width).transform_literal(&input)?; + assert_eq!(result, Some(expected), "width: {width}, input: {input:?}"); + } + Ok(()) + } + #[test] fn test_literal_int() { let input = Datum::int(1); @@ -831,12 +984,52 @@ mod test { #[test] fn test_decimal_literal() { - let input = Datum::decimal(decimal_new(1065, 0)).unwrap(); - let res = super::Truncate::new(50) - .transform_literal(&input) - .unwrap() - .unwrap(); - assert_eq!(res, Datum::decimal(decimal_new(1050, 0)).unwrap(),); + for (precision, scale, value, expected) in [ + (9, 0, 1065, 1050), + (9, 2, 1065, 1050), + (9, 2, -1065, -1100), + (9, 2, 1050, 1050), + (9, 2, 0, 0), + (9, 5, 1065, 1050), + (38, 2, 1065, 1050), + ] { + let data_type = Decimal { precision, scale }; + let input = Datum::new(data_type.clone(), PrimitiveLiteral::Int128(value)); + let res = super::Truncate::new(50) + .transform_literal(&input) + .unwrap() + .unwrap(); + assert_eq!( + res, + Datum::new(data_type, PrimitiveLiteral::Int128(expected)), + "input: {input:?}" + ); + } + } + + #[test] + fn test_binary_literal() { + for (value, width, expected) in [ + (vec![0x00, 0xff, 0x80, 0x01], 2, vec![0x00, 0xff]), + (vec![0x00, 0xff], 2, vec![0x00, 0xff]), + (vec![0x00, 0xff], 4, vec![0x00, 0xff]), + (vec![], 2, vec![]), + ] { + let input = Datum::binary(value); + let res = super::Truncate::new(width) + .transform_literal(&input) + .unwrap() + .unwrap(); + assert_eq!(res, Datum::binary(expected)); + } + } + + #[test] + fn test_fixed_literal_is_unsupported() { + let error = super::Truncate::new(2) + .transform_literal(&Datum::fixed(vec![0x00, 0xff, 0x80])) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::FeatureUnsupported); } #[test] From 1270e329b517a88bdb8eb641b4a59f45bdc0e872 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Wed, 14 Oct 2026 03:42:36 +0000 Subject: [PATCH 62/63] chore(deps): Bump quinn-proto and event-listener to fix RUSTSEC advisories (#3425) chore(deps): Bump quinn-proto to fix RUSTSEC advisories Update Cargo.lock to pick up quinn-proto 0.11.19, which fixes RUSTSEC-2026-0338/0339/0340/0341/0342. Also bump event-listener (RUSTSEC-2026-0221, unsound) and the yanked chacha20 and spin releases so cargo audit runs clean. --- Cargo.lock | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3694ad2e2..582309e632 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1303,9 +1303,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.3.0", @@ -1426,15 +1426,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "const-oid" version = "0.9.6" @@ -2029,11 +2020,10 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -2123,7 +2113,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin 0.9.8", + "spin 0.9.9", ] [[package]] @@ -3309,7 +3299,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin 0.9.9", ] [[package]] @@ -4712,9 +4702,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", @@ -5883,9 +5873,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] From 47e4e761406426c84412cd80b2256fadb204eb52 Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Thu, 15 Oct 2026 10:29:11 +0200 Subject: [PATCH 63/63] Converge pipeline.rs's open/schema-resolution boundary with upstream's shape This fork's open_parquet_stream_builder bundled "open the file" and "resolve schema + install virtual columns" into one function taking virtual_columns as an input parameter, so it could be used inside a single futures::join! with concurrent delete-file loading. Upstream keeps these as two steps: a plain open_parquet_file, then a separate configure_arrow_reader_metadata call afterward (hardcoded to only ever install its own row-number field). Split open_parquet_stream_builder back into those same two functions: open_parquet_file (now bytes_read: Option<&Arc>, so it also serves the three call sites that need virtual columns) and a generalized configure_arrow_reader_metadata that keeps accepting an arbitrary virtual_columns: Vec> rather than hardcoding row-number, preserving this fork's flexibility. The schema-resolution logic inside configure_arrow_reader_metadata is unchanged from the single-rebuild fix in the previous commit; only the function boundary moved. Updates all four callers: FileScanTaskReader::process (pipeline.rs), process_incremental_append_task and process_equality_delete_task (arrow/incremental.rs), and delete_file_loader.rs's simpler read path (now wrapping its always-on counter in Some(..)). Each non-delete-loader caller now computes has_missing_field_ids itself and builds the ParquetRecordBatchStreamBuilder directly, matching upstream's process() exactly; the futures::join! concurrency with delete-file loading in FileScanTaskReader::process is preserved unchanged, just wrapping the narrower open_parquet_file call instead of the combined one. Rationale: pipeline.rs is the single most frequently touched file across upstream syncs (4 of the last 62 commits touched it, more than any other file in this merge). Matching upstream's function boundary here should reduce conflict surface on the next sync, without touching the downstream row-id/spec-id/last-updated-sequence-number/bloom-filter logic, which was already independently convergent with upstream's own (both inherited it from shared ancestry, not fork-exclusive as earlier analysis assumed). Verified: cargo test -p iceberg --lib (1996 passed, 0 failed), cargo fmt --check and cargo clippy --all-targets -D warnings clean. Co-Authored-By: Claude Sonnet 5 --- .../iceberg/src/arrow/delete_file_loader.rs | 2 +- crates/iceberg/src/arrow/incremental.rs | 67 ++++-- crates/iceberg/src/arrow/reader/pipeline.rs | 192 ++++++++---------- 3 files changed, 135 insertions(+), 126 deletions(-) diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index ec61710a42..6bbdcd9e6d 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -80,7 +80,7 @@ impl BasicDeleteFileLoader { &self.file_io, file_size_in_bytes, parquet_read_options, - self.scan_metrics.bytes_read_counter(), + Some(self.scan_metrics.bytes_read_counter()), key_metadata, ) .await?; diff --git a/crates/iceberg/src/arrow/incremental.rs b/crates/iceberg/src/arrow/incremental.rs index df3a7314c4..88186133c2 100644 --- a/crates/iceberg/src/arrow/incremental.rs +++ b/crates/iceberg/src/arrow/incremental.rs @@ -23,6 +23,7 @@ use arrow_schema::Schema as ArrowSchema; use futures::channel::mpsc::channel; use futures::stream::select; use futures::{Stream, StreamExt, TryStreamExt}; +use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder}; use crate::arrow::reader::{ PRUNABLE_METADATA_FIELDS, ParquetReadOptions, process_record_batch_stream, @@ -113,20 +114,39 @@ async fn process_incremental_append_task( virtual_columns.push(Arc::clone(row_pos_field())); } - let (builder, has_missing_field_ids) = ArrowReader::open_parquet_stream_builder( + let (file_reader, arrow_metadata) = ArrowReader::open_parquet_file( &base.data_file_path, + &file_io, base.file_size_in_bytes, - file_io, parquet_read_options, - virtual_columns, - batch_size, - None, // name_mapping not yet supported in incremental scan - Some(Arc::clone(scan_metrics.bytes_read_counter())), - Some(&base.schema), + Some(scan_metrics.bytes_read_counter()), base.key_metadata.as_deref(), ) .await?; + // Check if Parquet file has embedded field IDs. + // Corresponds to Java's ParquetSchemaUtil.hasIds() + let has_missing_field_ids = arrow_metadata + .schema() + .fields() + .iter() + .next() + .is_some_and(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none()); + + let arrow_metadata = ArrowReader::configure_arrow_reader_metadata( + arrow_metadata, + has_missing_field_ids, + None, // name_mapping not yet supported in incremental scan + &base.schema, + virtual_columns, + )?; + + let mut builder = + ParquetRecordBatchStreamBuilder::new_with_metadata(file_reader, arrow_metadata); + if let Some(batch_size) = batch_size { + builder = builder.with_batch_size(batch_size); + } + let builder = ArrowReader::apply_parquet_filters( builder, base.start, @@ -248,20 +268,39 @@ async fn process_equality_delete_task( .combined_predicate .bind(task.schema_ref(), task.base.case_sensitive)?; - let (builder, has_missing_field_ids) = ArrowReader::open_parquet_stream_builder( + let (file_reader, arrow_metadata) = ArrowReader::open_parquet_file( &task.base.data_file_path, + &file_io, task.base.file_size_in_bytes, - file_io, parquet_read_options, - vec![Arc::clone(row_pos_field())], - batch_size, - None, // name_mapping not yet supported in incremental scan - Some(Arc::clone(scan_metrics.bytes_read_counter())), - Some(&task.base.schema), + Some(scan_metrics.bytes_read_counter()), task.base.key_metadata.as_deref(), ) .await?; + // Check if Parquet file has embedded field IDs. + // Corresponds to Java's ParquetSchemaUtil.hasIds() + let has_missing_field_ids = arrow_metadata + .schema() + .fields() + .iter() + .next() + .is_some_and(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none()); + + let arrow_metadata = ArrowReader::configure_arrow_reader_metadata( + arrow_metadata, + has_missing_field_ids, + None, // name_mapping not yet supported in incremental scan + &task.base.schema, + vec![Arc::clone(row_pos_field())], + )?; + + let mut builder = + ParquetRecordBatchStreamBuilder::new_with_metadata(file_reader, arrow_metadata); + if let Some(batch_size) = batch_size { + builder = builder.with_batch_size(batch_size); + } + let builder = ArrowReader::apply_parquet_filters( builder, task.base.start, diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 259e34eb1f..403707adb3 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -209,16 +209,12 @@ impl FileScanTaskReader { } // Concurrently open the Parquet file and start loading delete files. - let open_fut = ArrowReader::open_parquet_stream_builder( + let open_fut = ArrowReader::open_parquet_file( task.data_file_path(), + &self.file_io, task.file_size_in_bytes(), - self.file_io.clone(), parquet_read_options, - virtual_columns, - self.batch_size, - task.name_mapping().map(|nm| nm.as_ref()), - Some(Arc::clone(self.scan_metrics.bytes_read_counter())), - Some(task.schema()), + Some(self.scan_metrics.bytes_read_counter()), task.key_metadata(), ); let delete_filter_rx = self @@ -228,12 +224,35 @@ impl FileScanTaskReader { let (open_result, delete_filter) = futures::join!(open_fut, async { delete_filter_rx.await.unwrap() }); - let (builder, has_missing_field_ids) = open_result?; + let (file_reader, arrow_metadata) = open_result?; let delete_filter = delete_filter?; + // Check if Parquet file has embedded field IDs. + // Corresponds to Java's ParquetSchemaUtil.hasIds() + let has_missing_field_ids = arrow_metadata + .schema() + .fields() + .iter() + .next() + .is_some_and(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none()); + + let arrow_metadata = ArrowReader::configure_arrow_reader_metadata( + arrow_metadata, + has_missing_field_ids, + task.name_mapping().map(|nm| nm.as_ref()), + task.schema(), + virtual_columns, + )?; + + let mut builder = + ParquetRecordBatchStreamBuilder::new_with_metadata(file_reader, arrow_metadata); + if let Some(batch_size) = self.batch_size { + builder = builder.with_batch_size(batch_size); + } + // Position-based fallback applies only when the file has no embedded field IDs // AND no name mapping is available. With a name mapping, field IDs are assigned - // to the Arrow schema by `open_parquet_stream_builder`, and projection/predicate + // to the Arrow schema by `configure_arrow_reader_metadata`, and projection/predicate // planning must use them (see #2403). let use_position_fallback = has_missing_field_ids && task.name_mapping().is_none(); @@ -357,21 +376,29 @@ impl ArrowReader { Ok(result) } - /// Opens a Parquet file and loads its metadata, wrapping the reader with - /// [`CountingFileRead`] so all I/O is accumulated into `bytes_read`. + /// Opens a Parquet file and loads its metadata. When `bytes_read` is `Some`, wraps the + /// reader with [`CountingFileRead`] so all I/O is accumulated into the provided counter. + /// + /// This is the async, I/O-bound phase shared by every reading path. Callers that have + /// background work to overlap (e.g. delete-file loading) should run this concurrently + /// with that work using [`futures::join!`], then resolve the schema with + /// [`Self::configure_arrow_reader_metadata`]. pub(crate) async fn open_parquet_file( data_file_path: &str, file_io: &FileIO, file_size_in_bytes: u64, parquet_read_options: ParquetReadOptions, - bytes_read: &Arc, + bytes_read: Option<&Arc>, key_metadata: Option<&[u8]>, ) -> Result<(ArrowFileReader, ArrowReaderMetadata)> { let parquet_file = file_io.new_input(data_file_path)?; - let counting_reader = - CountingFileRead::new(parquet_file.reader().await?, Arc::clone(bytes_read)); + let raw_reader = parquet_file.reader().await?; + let boxed_reader: Box = match bytes_read { + Some(counter) => Box::new(CountingFileRead::new(raw_reader, Arc::clone(counter))), + None => Box::new(raw_reader), + }; Self::build_parquet_reader( - Box::new(counting_reader), + boxed_reader, file_size_in_bytes, parquet_read_options, key_metadata, @@ -404,81 +431,35 @@ impl ArrowReader { Ok((reader, arrow_metadata)) } - /// Opens a Parquet file, resolves its schema (name-mapping / field-ID fallback), and - /// applies the batch size. Returns `(builder, has_missing_field_ids)`. - /// - /// This is the async phase shared by every reading path. Callers that have background - /// work to overlap (e.g. delete-file loading) can run this concurrently with that work - /// using [`futures::join!`], then pass the result to [`Self::apply_parquet_filters`]. + /// Resolves the schema (field-id assignment, then INT96 coercion) and applies any + /// virtual columns, rebuilding the Arrow reader metadata at most once. /// /// Implements the three-branch schema resolution strategy matching Java's `ReadConf` constructor: /// - Branch 1: file has embedded field IDs → trust them, use as-is /// - Branch 2: name_mapping present → apply name mapping to assign correct Iceberg field IDs /// - Branch 3: no name mapping → assign fallback position-based IDs /// - /// When `iceberg_schema` is `Some`, INT96 timestamp columns are coerced to the resolution - /// specified by the Iceberg schema before building the stream reader. - /// - /// When `bytes_read` is `Some`, wraps the file reader with [`CountingFileRead`] so all - /// I/O bytes are accumulated into the provided counter. - /// - /// When `key_metadata` is `Some`, the file is opened with Parquet Modular Encryption - /// decryption properties derived from it. - #[allow(clippy::too_many_arguments)] - pub(crate) async fn open_parquet_stream_builder( - data_file_path: &str, - file_size_in_bytes: u64, - file_io: FileIO, - parquet_read_options: ParquetReadOptions, - virtual_columns: Vec>, - batch_size: Option, + /// Field-id resolution and INT96 coercion both operate on a plain schema value, with + /// virtual columns applied once at the very end in a single rebuild. Two bugs are + /// avoided by not interleaving virtual columns into the earlier steps: + /// 1. A schema that already carries a virtual column's field confuses + /// `coerce_int96_timestamps`, which expects the arrow schema to line up + /// field-for-field with the real Iceberg schema. + /// 2. A fresh `ArrowReaderOptions` doesn't carry over a previous rebuild's + /// virtual-column registration, only its schema shape -- so a schema that merely + /// *looks like* it has the virtual column, without the reader being told to + /// synthesize it, fails at read time ("virtual column ... is not present in + /// source batch"). + pub(crate) fn configure_arrow_reader_metadata( + arrow_metadata: ArrowReaderMetadata, + missing_field_ids: bool, name_mapping: Option<&NameMapping>, - bytes_read: Option>, - iceberg_schema: Option<&crate::spec::Schema>, - key_metadata: Option<&[u8]>, - ) -> Result<(ParquetRecordBatchStreamBuilder, bool)> { - let parquet_file = file_io.new_input(data_file_path)?; - let raw_reader = parquet_file.reader().await?; - let boxed_reader: Box = if let Some(counter) = bytes_read { - Box::new(CountingFileRead::new(raw_reader, counter)) - } else { - Box::new(raw_reader) - }; - let (file_reader, arrow_metadata) = Self::build_parquet_reader( - boxed_reader, - file_size_in_bytes, - parquet_read_options, - key_metadata, - ) - .await?; - - // Check if Parquet file has embedded field IDs. - // Corresponds to Java's ParquetSchemaUtil.hasIds() - let has_missing_field_ids = arrow_metadata - .schema() - .fields() - .iter() - .next() - .is_some_and(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none()); + iceberg_schema: &crate::spec::Schema, + virtual_columns: Vec>, + ) -> Result { + let mut schema_changed = missing_field_ids; - // Field-id resolution and INT96 coercion both operate on a plain schema value, with - // virtual columns applied once at the very end in a single rebuild. Two bugs are - // avoided by not interleaving virtual columns into the earlier steps: - // 1. A schema that already carries a virtual column's field confuses - // `coerce_int96_timestamps`, which expects the arrow schema to line up - // field-for-field with the real Iceberg schema. - // 2. A fresh `ArrowReaderOptions` doesn't carry over a previous rebuild's - // virtual-column registration, only its schema shape -- so a schema that merely - // *looks like* it has the virtual column, without the reader being told to - // synthesize it, fails at read time ("virtual column ... is not present in - // source batch"). - let mut schema_changed = has_missing_field_ids; - - // Three-branch schema resolution strategy matching Java's ReadConf constructor. - // - // When Parquet files lack field IDs (e.g., Hive/Spark migrations via add_files), - // we must assign field IDs BEFORE reading data to enable correct column projection. - let resolved_schema = if has_missing_field_ids { + let resolved_schema = if missing_field_ids { // Parquet file lacks field IDs - must assign them before reading. if let Some(nm) = name_mapping { // Branch 2: Apply name mapping to assign correct Iceberg field IDs. @@ -496,9 +477,7 @@ impl ArrowReader { // Coerce INT96 timestamp columns to the resolution specified by the Iceberg schema. // This must happen before building the stream reader to avoid i64 overflow in arrow-rs. - let resolved_schema = match iceberg_schema - .and_then(|schema| coerce_int96_timestamps(&resolved_schema, schema)) - { + let resolved_schema = match coerce_int96_timestamps(&resolved_schema, iceberg_schema) { Some(coerced) => { schema_changed = true; coerced @@ -506,38 +485,29 @@ impl ArrowReader { None => resolved_schema, }; - let arrow_metadata = if schema_changed || !virtual_columns.is_empty() { - let mut options = ArrowReaderOptions::new().with_schema(Arc::clone(&resolved_schema)); - if !virtual_columns.is_empty() { - options = options.with_virtual_columns(virtual_columns)?; - } - ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err( - |e| { - Error::new( - ErrorKind::Unexpected, - format!( - "Failed to create ArrowReaderMetadata with resolved schema: {resolved_schema}" - ), - ) - .with_source(e) - }, - )? - } else { - arrow_metadata - }; - - let mut builder = - ParquetRecordBatchStreamBuilder::new_with_metadata(file_reader, arrow_metadata); + if !schema_changed && virtual_columns.is_empty() { + return Ok(arrow_metadata); + } - if let Some(batch_size) = batch_size { - builder = builder.with_batch_size(batch_size); + let mut options = ArrowReaderOptions::new().with_schema(Arc::clone(&resolved_schema)); + if !virtual_columns.is_empty() { + options = options.with_virtual_columns(virtual_columns)?; } - Ok((builder, has_missing_field_ids)) + ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!( + "Failed to create ArrowReaderMetadata with resolved schema: {resolved_schema}" + ), + ) + .with_source(e) + }) } /// Applies all row-level and row-group-level filters to a builder returned by - /// [`Self::open_parquet_stream_builder`]. + /// building a [`ParquetRecordBatchStreamBuilder`] from [`Self::open_parquet_file`] and + /// [`Self::configure_arrow_reader_metadata`]'s result. /// /// Handles byte-range row group pruning, predicate row filtering (with optional /// projection), and positional-delete row selection.