From a5f162fed26d7b63bcc25ec8ed08a00d48ab229e Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Thu, 27 Aug 2026 11:26:48 -0700 Subject: [PATCH 01/74] perf(arrow): build repeated constant string/binary columns without per-row clones (#3080) * perf(arrow): build repeated constant string/binary columns without per-row clones create_primitive_array_repeated built a throwaway `vec![value.clone(); num_rows]` for the Utf8/Binary/LargeBinary/FixedSizeBinary arms before handing it to the array constructor. For strings that clones the value into num_rows separate heap allocations per batch; the binary arms allocate a throwaway intermediate Vec. Stream the single value straight into the Arrow buffer via `from_iter_values` with `std::iter::repeat_n` instead. Benchmarks (release, throwaway harness, old vs new toggled on this file only): - leaf create_primitive_array_repeated (Utf8, 8192-row batch): 23.0 -> 4.1 ns/row (5.6x) - full scan select(x, _partition) over a real manifest + 262k-row parquet file, string-partitioned: 20.9 -> 5.7 ns/row (3.7x) Note that the gain is allocation-counts and scales with how much of the projection is the string partition column. Fixes #3079 * Feedback --- crates/iceberg/src/arrow/value.rs | 133 ++++++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 15 deletions(-) diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index d22e565a4a..a19771a3d0 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -864,23 +864,38 @@ pub(crate) fn create_primitive_array_repeated( (DataType::Float64, Some(PrimitiveLiteral::Double(value))) => { Arc::new(Float64Array::from(vec![value.0; num_rows])) } - (DataType::Utf8, Some(PrimitiveLiteral::String(value))) => { - Arc::new(StringArray::from(vec![value.clone(); num_rows])) - } - (DataType::Binary, Some(PrimitiveLiteral::Binary(value))) => { - Arc::new(BinaryArray::from_vec(vec![value; num_rows])) - } - (DataType::LargeBinary, Some(PrimitiveLiteral::Binary(value))) => { - Arc::new(LargeBinaryArray::from_vec(vec![value; num_rows])) - } + (DataType::Utf8, Some(PrimitiveLiteral::String(value))) => Arc::new( + StringArray::from_iter_values(std::iter::repeat_n(value.as_str(), num_rows)), + ), + (DataType::Binary, Some(PrimitiveLiteral::Binary(value))) => Arc::new( + BinaryArray::from_iter_values(std::iter::repeat_n(value.as_slice(), num_rows)), + ), + (DataType::LargeBinary, Some(PrimitiveLiteral::Binary(value))) => Arc::new( + LargeBinaryArray::from_iter_values(std::iter::repeat_n(value.as_slice(), num_rows)), + ), (DataType::FixedSizeBinary(len), Some(PrimitiveLiteral::Binary(value))) => { - let repeated: Vec<&[u8]> = vec![value.as_slice(); num_rows]; - Arc::new(FixedSizeBinaryArray::try_from_iter(repeated.into_iter()).map_err(|e| { - Error::new( + // try_from_iter infers the width from the data, never from `len`, so a + // wrong-width literal would otherwise produce a FixedSizeBinary of the + // wrong width instead of erroring. + if value.len() != *len as usize { + return Err(Error::new( ErrorKind::DataInvalid, - format!("Failed to create FixedSizeBinary({len}) array: {e}"), - ) - })?) + format!( + "FixedSizeBinary literal length {} does not match declared width {len}", + value.len() + ), + )); + } + + Arc::new( + FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n(value.as_slice(), num_rows)) + .map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Failed to create FixedSizeBinary({len}) array: {e}"), + ) + })?, + ) } (DataType::Time64(TimeUnit::Microsecond), Some(PrimitiveLiteral::Long(value))) => { Arc::new(Time64MicrosecondArray::from(vec![*value; num_rows])) @@ -1914,4 +1929,92 @@ mod test { assert_eq!(array.data_type(), &target_type); assert_eq!(array.len(), num_rows); } + + #[test] + fn test_create_string_and_binary_arrays_repeated() { + let text = "partition-value-2026"; + let bytes: Vec = vec![0xDE, 0xAD, 0xBE, 0xEF]; + let num_rows = 4; + + let utf8 = create_primitive_array_repeated( + &DataType::Utf8, + &Some(PrimitiveLiteral::String(text.to_string())), + num_rows, + ) + .unwrap(); + let utf8 = utf8.as_any().downcast_ref::().unwrap(); + assert_eq!(utf8.len(), num_rows); + assert!((0..num_rows).all(|i| utf8.value(i) == text)); + + let binary = create_primitive_array_repeated( + &DataType::Binary, + &Some(PrimitiveLiteral::Binary(bytes.clone())), + num_rows, + ) + .unwrap(); + let binary = binary.as_any().downcast_ref::().unwrap(); + assert_eq!(binary.len(), num_rows); + assert!((0..num_rows).all(|i| binary.value(i) == bytes.as_slice())); + + let large = create_primitive_array_repeated( + &DataType::LargeBinary, + &Some(PrimitiveLiteral::Binary(bytes.clone())), + num_rows, + ) + .unwrap(); + let large = large.as_any().downcast_ref::().unwrap(); + assert_eq!(large.len(), num_rows); + assert!((0..num_rows).all(|i| large.value(i) == bytes.as_slice())); + + let fixed = create_primitive_array_repeated( + &DataType::FixedSizeBinary(bytes.len() as i32), + &Some(PrimitiveLiteral::Binary(bytes.clone())), + num_rows, + ) + .unwrap(); + let fixed = fixed + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(fixed.len(), num_rows); + assert!((0..num_rows).all(|i| fixed.value(i) == bytes.as_slice())); + } + + #[test] + fn test_create_string_array_repeated_empty() { + // num_rows == 0 must produce an empty (not one-element) array. + let array = create_primitive_array_repeated( + &DataType::Utf8, + &Some(PrimitiveLiteral::String("x".to_string())), + 0, + ) + .unwrap(); + assert_eq!(array.len(), 0); + } + + #[test] + fn test_create_fixed_size_binary_repeated_wrong_width_errors() { + // A literal whose length differs from the declared width must error rather + // than silently producing a FixedSizeBinary array of the literal's width. + let err = create_primitive_array_repeated( + &DataType::FixedSizeBinary(4), + &Some(PrimitiveLiteral::Binary(vec![0x01, 0x02, 0x03])), + 2, + ) + .unwrap_err(); + assert!(err.to_string().contains("does not match declared width")); + } + + #[test] + fn test_create_fixed_size_binary_repeated_empty_errors() { + // arrow's try_from_iter cannot infer the fixed width from an empty iterator, + // so num_rows == 0 over a fixed[n] column surfaces as an error rather than an + // empty fixed[n] array. + let result = create_primitive_array_repeated( + &DataType::FixedSizeBinary(4), + &Some(PrimitiveLiteral::Binary(vec![0xDE, 0xAD, 0xBE, 0xEF])), + 0, + ); + assert!(result.is_err()); + } } From 4382ebe12fd746b6796d6113497a9e3d6c6ec975 Mon Sep 17 00:00:00 2001 From: Daniel Carl Jones Date: Thu, 27 Aug 2026 23:32:05 +0100 Subject: [PATCH 02/74] chore: Update MSRV to Rust 1.95 (#3083) --- Cargo.toml | 2 +- crates/catalog/glue/src/catalog.rs | 2 +- crates/catalog/glue/src/schema.rs | 2 +- crates/iceberg/src/arrow/schema.rs | 2 +- crates/iceberg/src/arrow/value.rs | 4 +- .../expr/visitors/bound_predicate_visitor.rs | 6 +-- .../src/expr/visitors/expression_evaluator.rs | 34 ++++++++------ .../src/expr/visitors/manifest_evaluator.rs | 8 ++-- .../src/expr/visitors/page_index_evaluator.rs | 15 ++++--- .../src/expr/visitors/predicate_visitor.rs | 6 +-- .../visitors/row_group_metrics_evaluator.rs | 45 ++++++++++--------- .../expr/visitors/strict_metrics_evaluator.rs | 4 +- crates/iceberg/src/spec/manifest/mod.rs | 8 ++-- crates/iceberg/src/spec/manifest_list/mod.rs | 4 +- crates/iceberg/src/spec/schema/mod.rs | 4 +- crates/iceberg/src/spec/values/datum.rs | 2 +- crates/iceberg/src/spec/values/serde.rs | 2 +- crates/iceberg/src/spec/values/tests.rs | 28 ++++++------ .../src/writer/file_writer/parquet_writer.rs | 24 +++++----- crates/sqllogictest/src/schedule.rs | 2 +- rust-toolchain.toml | 2 +- 21 files changed, 110 insertions(+), 96 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 38dd546f94..cea9ede78a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ version = "0.10.0" license = "Apache-2.0" repository = "https://github.com/apache/iceberg-rust" # Check the MSRV policy in README.md before changing this -rust-version = "1.94" +rust-version = "1.95" [workspace.lints.rust] unused_qualifications = "deny" diff --git a/crates/catalog/glue/src/catalog.rs b/crates/catalog/glue/src/catalog.rs index 48fc6bb01a..721fbb548c 100644 --- a/crates/catalog/glue/src/catalog.rs +++ b/crates/catalog/glue/src/catalog.rs @@ -539,7 +539,7 @@ impl Catalog for GlueCatalog { if !resp.table_list().is_empty() { return Err(Error::new( ErrorKind::DataInvalid, - format!("Database with name: {} is not empty", &db_name), + format!("Database with name: {} is not empty", db_name), )); } diff --git a/crates/catalog/glue/src/schema.rs b/crates/catalog/glue/src/schema.rs index 59d51f3dc1..6a33a9cba2 100644 --- a/crates/catalog/glue/src/schema.rs +++ b/crates/catalog/glue/src/schema.rs @@ -103,7 +103,7 @@ impl SchemaVisitor for GlueSchemaBuilder { fn field(&mut self, field: &iceberg::spec::NestedFieldRef, value: String) -> Result { if self.is_inside_struct() { - return Ok(format!("{}:{}", field.name, &value)); + return Ok(format!("{}:{}", field.name, value)); } let parameters = HashMap::from([ diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index ae5cbb8173..349ada859e 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -2388,7 +2388,7 @@ mod tests { assert_eq!(array.value(0), 42); } { - let datum = Datum::float(42.42); + let datum = Datum::float(42.42_f32); let arrow_datum = get_arrow_datum(&datum).unwrap(); let (array, is_scalar) = arrow_datum.get(); let array = array.as_any().downcast_ref::().unwrap(); diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index a19771a3d0..34b4336ef2 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -1201,7 +1201,7 @@ mod test { Some(Literal::bool(true)), Some(Literal::int(3)), Some(Literal::long(5)), - Some(Literal::float(1.1)), + Some(Literal::float(1.1_f32)), Some(Literal::double(3.3)), Some(Literal::decimal(1000)), Some(Literal::date(18628)), @@ -1215,7 +1215,7 @@ mod test { Some(Literal::bool(false)), Some(Literal::int(4)), Some(Literal::long(6)), - Some(Literal::float(2.2)), + Some(Literal::float(2.2_f32)), Some(Literal::double(4.4)), Some(Literal::decimal(2000)), Some(Literal::date(18629)), diff --git a/crates/iceberg/src/expr/visitors/bound_predicate_visitor.rs b/crates/iceberg/src/expr/visitors/bound_predicate_visitor.rs index 22f6d08ad3..1d7816dcf7 100644 --- a/crates/iceberg/src/expr/visitors/bound_predicate_visitor.rs +++ b/crates/iceberg/src/expr/visitors/bound_predicate_visitor.rs @@ -185,7 +185,7 @@ pub(crate) fn visit( PredicateOperator::IsNan => visitor.is_nan(expr.term(), predicate), PredicateOperator::NotNan => visitor.not_nan(expr.term(), predicate), op => { - panic!("Unexpected op for unary predicate: {}", &op) + panic!("Unexpected op for unary predicate: {}", op) } }, BoundPredicate::Binary(expr) => { @@ -209,7 +209,7 @@ pub(crate) fn visit( visitor.not_starts_with(reference, literal, predicate) } op => { - panic!("Unexpected op for binary predicate: {}", &op) + panic!("Unexpected op for binary predicate: {}", op) } } } @@ -220,7 +220,7 @@ pub(crate) fn visit( PredicateOperator::In => visitor.r#in(reference, literals, predicate), PredicateOperator::NotIn => visitor.not_in(reference, literals, predicate), op => { - panic!("Unexpected op for set predicate: {}", &op) + panic!("Unexpected op for set predicate: {}", op) } } } diff --git a/crates/iceberg/src/expr/visitors/expression_evaluator.rs b/crates/iceberg/src/expr/visitors/expression_evaluator.rs index 570c409502..122f195897 100644 --- a/crates/iceberg/src/expr/visitors/expression_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/expression_evaluator.rs @@ -330,7 +330,7 @@ mod tests { } fn create_data_file_float() -> DataFile { - let partition = Struct::from_iter([Some(Literal::float(1.0))]); + let partition = Struct::from_iter([Some(Literal::float(1.0_f32))]); DataFile { content: DataContentType::Data, @@ -393,12 +393,12 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::LessThan, Reference::new("a"), - Datum::float(1.0), + Datum::float(1.0_f32), )) .or(Predicate::Binary(BinaryExpression::new( PredicateOperator::GreaterThanOrEq, Reference::new("a"), - Datum::float(0.4), + Datum::float(0.4_f32), ))) .bind(schema.clone(), case_sensitive)?; @@ -422,12 +422,12 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::LessThan, Reference::new("a"), - Datum::float(1.1), + Datum::float(1.1_f32), )) .and(Predicate::Binary(BinaryExpression::new( PredicateOperator::GreaterThanOrEq, Reference::new("a"), - Datum::float(0.4), + Datum::float(0.4_f32), ))) .bind(schema.clone(), case_sensitive)?; @@ -451,7 +451,11 @@ mod tests { let predicate = Predicate::Set(SetExpression::new( PredicateOperator::NotIn, Reference::new("a"), - FnvHashSet::from_iter([Datum::float(0.9), Datum::float(1.2), Datum::float(2.4)]), + FnvHashSet::from_iter([ + Datum::float(0.9_f32), + Datum::float(1.2_f32), + Datum::float(2.4_f32), + ]), )) .bind(schema.clone(), case_sensitive)?; @@ -475,7 +479,11 @@ mod tests { let predicate = Predicate::Set(SetExpression::new( PredicateOperator::In, Reference::new("a"), - FnvHashSet::from_iter([Datum::float(1.0), Datum::float(1.2), Datum::float(2.4)]), + FnvHashSet::from_iter([ + Datum::float(1.0_f32), + Datum::float(1.2_f32), + Datum::float(2.4_f32), + ]), )) .bind(schema.clone(), case_sensitive)?; @@ -547,7 +555,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::NotEq, Reference::new("a"), - Datum::float(0.9), + Datum::float(0.9_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -571,7 +579,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::Eq, Reference::new("a"), - Datum::float(1.0), + Datum::float(1.0_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -595,7 +603,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::GreaterThanOrEq, Reference::new("a"), - Datum::float(1.0), + Datum::float(1.0_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -619,7 +627,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::GreaterThan, Reference::new("a"), - Datum::float(0.9), + Datum::float(0.9_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -643,7 +651,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::LessThanOrEq, Reference::new("a"), - Datum::float(1.0), + Datum::float(1.0_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -667,7 +675,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::LessThan, Reference::new("a"), - Datum::float(1.1), + Datum::float(1.1_f32), )) .bind(schema.clone(), case_sensitive)?; diff --git a/crates/iceberg/src/expr/visitors/manifest_evaluator.rs b/crates/iceberg/src/expr/visitors/manifest_evaluator.rs index 243b317570..53dbafc065 100644 --- a/crates/iceberg/src/expr/visitors/manifest_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/manifest_evaluator.rs @@ -612,8 +612,8 @@ mod test { FieldSummary { contains_null: true, contains_nan: None, - lower_bound: Some(Datum::float(0.0).to_bytes().unwrap()), - upper_bound: Some(Datum::float(20.0).to_bytes().unwrap()), + lower_bound: Some(Datum::float(0.0_f32).to_bytes().unwrap()), + upper_bound: Some(Datum::float(20.0_f32).to_bytes().unwrap()), }, // all_nulls_double FieldSummary { @@ -647,8 +647,8 @@ mod test { FieldSummary { contains_null: false, contains_nan: Some(false), - lower_bound: Some(Datum::float(0.0).to_bytes().unwrap()), - upper_bound: Some(Datum::float(20.0).to_bytes().unwrap()), + lower_bound: Some(Datum::float(0.0_f32).to_bytes().unwrap()), + upper_bound: Some(Datum::float(20.0_f32).to_bytes().unwrap()), }, // all_nulls_missing_nan_float FieldSummary { diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index f30760d5fc..bd901eda5f 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -941,7 +941,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let row_group_metadata = metadata.row_group(0); @@ -1103,7 +1103,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .less_than(Datum::float(5.0)) + .less_than(Datum::float(5.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( @@ -1133,7 +1133,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(5.0)) + .equal_to(Datum::float(5.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( @@ -1166,7 +1166,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .not_equal_to(Datum::float(5.0)) + .not_equal_to(Datum::float(5.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( @@ -1192,7 +1192,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .starts_with(Datum::float(5.0)) + .starts_with(Datum::float(5.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( @@ -1284,7 +1284,10 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in(std::iter::repeat_with(|| Datum::float(rng.random_range(0.0..10.0))).take(1000)) + .is_in( + std::iter::repeat_with(|| Datum::float(rng.random_range(0.0_f32..10.0_f32))) + .take(1000), + ) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( diff --git a/crates/iceberg/src/expr/visitors/predicate_visitor.rs b/crates/iceberg/src/expr/visitors/predicate_visitor.rs index 61671cc2cd..1fa94e7f80 100644 --- a/crates/iceberg/src/expr/visitors/predicate_visitor.rs +++ b/crates/iceberg/src/expr/visitors/predicate_visitor.rs @@ -169,7 +169,7 @@ pub(crate) fn visit(visitor: &mut V, predicate: &Predicate) PredicateOperator::IsNan => visitor.is_nan(expr.term(), predicate), PredicateOperator::NotNan => visitor.not_nan(expr.term(), predicate), op => { - panic!("Unexpected op for unary predicate: {}", &op) + panic!("Unexpected op for unary predicate: {}", op) } }, Predicate::Binary(expr) => { @@ -193,7 +193,7 @@ pub(crate) fn visit(visitor: &mut V, predicate: &Predicate) visitor.not_starts_with(reference, literal, predicate) } op => { - panic!("Unexpected op for binary predicate: {}", &op) + panic!("Unexpected op for binary predicate: {}", op) } } } @@ -204,7 +204,7 @@ pub(crate) fn visit(visitor: &mut V, predicate: &Predicate) PredicateOperator::In => visitor.r#in(reference, literals, predicate), PredicateOperator::NotIn => visitor.not_in(reference, literals, predicate), op => { - panic!("Unexpected op for set predicate: {}", &op) + panic!("Unexpected op for set predicate: {}", op) } } } diff --git a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs index ad7e19f548..3a5a406e78 100644 --- a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs @@ -117,7 +117,7 @@ impl<'a> RowGroupMetricsEvaluator<'a> { ErrorKind::Unexpected, format!( "Could not find a field with id '{}' in the snapshot schema", - &field_id + field_id ), )); }; @@ -127,7 +127,7 @@ impl<'a> RowGroupMetricsEvaluator<'a> { ErrorKind::Unexpected, format!( "Could not determine the PrimitiveType for field id '{}'", - &field_id + field_id ), )); }; @@ -542,7 +542,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -564,7 +564,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -700,7 +700,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -760,7 +760,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -793,7 +793,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -826,7 +826,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -853,7 +853,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -886,7 +886,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -919,7 +919,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -952,7 +952,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -985,7 +985,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1018,7 +1018,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1051,7 +1051,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .not_equal_to(Datum::float(1.0)) + .not_equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1105,7 +1105,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .starts_with(Datum::float(1.0)) + .starts_with(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1636,7 +1636,10 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in(std::iter::repeat_with(|| Datum::float(rng.random_range(0.0..10.0))).take(1000)) + .is_in( + std::iter::repeat_with(|| Datum::float(rng.random_range(0.0_f32..10.0_f32))) + .take(1000), + ) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1698,7 +1701,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in([Datum::float(2.0), Datum::float(3.0)]) + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1725,7 +1728,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in([Datum::float(2.0), Datum::float(3.0)]) + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1758,7 +1761,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in([Datum::float(2.0), Datum::float(3.0)]) + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1791,7 +1794,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in([Datum::float(2.0), Datum::float(3.0)]) + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( diff --git a/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs index 0371f547f8..9fe8f216cb 100644 --- a/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs @@ -27,15 +27,15 @@ const ROWS_MUST_MATCH: Result = Ok(true); #[allow(dead_code)] const ROWS_MIGHT_NOT_MATCH: Result = Ok(false); -#[allow(dead_code)] /// Evaluates an `Expression` on a `DataFile` to test whether all rows in the file match. -/// +/// /// This evaluation is strict: it returns true if all rows in a file must match the expression. /// For example, if a file's ts column has min X and max Y, this evaluator will return true for ts /// < Y+1 but not for ts < Y-1. /// /// Files are passed to `eval(DataFile)`, which returns true if all rows in the file /// must contain matching rows and false if the file may contain rows that do not match. +#[allow(dead_code)] pub(crate) struct StrictMetricsEvaluator<'a> { data_file: &'a DataFile, } diff --git a/crates/iceberg/src/spec/manifest/mod.rs b/crates/iceberg/src/spec/manifest/mod.rs index 9156914aa1..f05eeae774 100644 --- a/crates/iceberg/src/spec/manifest/mod.rs +++ b/crates/iceberg/src/spec/manifest/mod.rs @@ -1034,7 +1034,7 @@ mod tests { partition: Struct::from_iter( vec![ Some(Literal::int(2021)), - Some(Literal::float(1.0)), + Some(Literal::float(1.0_f32)), Some(Literal::double(2.0)), ] ), @@ -1069,7 +1069,7 @@ mod tests { partition: Struct::from_iter( vec![ Some(Literal::int(1111)), - Some(Literal::float(15.5)), + Some(Literal::float(15.5_f32)), Some(Literal::double(25.5)), ] ), @@ -1197,11 +1197,11 @@ mod tests { assert_eq!( partitions[1].clone().lower_bound.unwrap(), - Datum::float(1.0).to_bytes().unwrap() + Datum::float(1.0_f32).to_bytes().unwrap() ); assert_eq!( partitions[1].clone().upper_bound.unwrap(), - Datum::float(15.5).to_bytes().unwrap() + Datum::float(15.5_f32).to_bytes().unwrap() ); assert!(partitions[1].clone().contains_null); assert_eq!(partitions[1].clone().contains_nan, Some(true)); diff --git a/crates/iceberg/src/spec/manifest_list/mod.rs b/crates/iceberg/src/spec/manifest_list/mod.rs index ae80eeb7f3..25dd5bb93e 100644 --- a/crates/iceberg/src/spec/manifest_list/mod.rs +++ b/crates/iceberg/src/spec/manifest_list/mod.rs @@ -196,7 +196,7 @@ mod test { existing_rows_count: Some(0), deleted_rows_count: Some(0), partitions: Some( - vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1).to_bytes().unwrap())}] + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1_f32).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1_f32).to_bytes().unwrap())}] ), key_metadata: None, first_row_id: None, @@ -314,7 +314,7 @@ mod test { existing_rows_count: Some(0), deleted_rows_count: Some(0), partitions: Some( - vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1).to_bytes().unwrap())}] + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1_f32).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1_f32).to_bytes().unwrap())}] ), key_metadata: None, first_row_id: Some(13), diff --git a/crates/iceberg/src/spec/schema/mod.rs b/crates/iceberg/src/spec/schema/mod.rs index 708f91d609..35af6f0f84 100644 --- a/crates/iceberg/src/spec/schema/mod.rs +++ b/crates/iceberg/src/spec/schema/mod.rs @@ -1240,8 +1240,8 @@ table { )]))), Some(Literal::List(vec![Some(Literal::Struct( crate::spec::Struct::from_iter(vec![ - Some(Literal::float(52.509_09)), - Some(Literal::float(-1.885_249)), + Some(Literal::float(52.509_09_f32)), + Some(Literal::float(-1.885_249_f32)), ]), ))])), Some(Literal::Struct(crate::spec::Struct::from_iter(vec![ diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index 51da5d4e34..d5eccd8328 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -456,7 +456,7 @@ impl Datum { ErrorKind::DataInvalid, format!( "PrimitiveLiteral Int128 must be PrimitiveType Decimal but got {}", - &self.r#type + self.r#type ), )); }; diff --git a/crates/iceberg/src/spec/values/serde.rs b/crates/iceberg/src/spec/values/serde.rs index 053acca8b0..10a815aa45 100644 --- a/crates/iceberg/src/spec/values/serde.rs +++ b/crates/iceberg/src/spec/values/serde.rs @@ -681,7 +681,7 @@ pub(crate) mod _serde { .ok_or_else(|| { invalid_err_with_reason( "record", - &format!("field {} is not exist", &field_name), + &format!("field {} is not exist", field_name), ) })?; let value = value.try_into(&field.field_type)?; diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index 8bf3311004..5ee4ac8d02 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -458,7 +458,7 @@ fn avro_bytes_long_from_int() { fn avro_bytes_float() { let bytes = vec![0u8, 0u8, 128u8, 63u8]; - check_avro_bytes_serde(bytes, Datum::float(1.0), &PrimitiveType::Float); + check_avro_bytes_serde(bytes, Datum::float(1.0_f32), &PrimitiveType::Float); } #[test] @@ -1077,7 +1077,7 @@ fn test_datum_ser_deser() { let datum = Datum::long(1); test_fn(datum); - let datum = Datum::float(1.0); + let datum = Datum::float(1.0_f32); test_fn(datum); let datum = Datum::float(0_f32); test_fn(datum); @@ -1200,7 +1200,7 @@ fn test_datum_double_convert_to_float() { let result = datum.to(&Primitive(PrimitiveType::Float)).unwrap(); - let expected = Datum::float(2.5); + let expected = Datum::float(2.5_f32); assert_eq!(result, expected); } @@ -1229,7 +1229,7 @@ fn test_datum_double_convert_to_float_below_min() { #[test] fn test_datum_float_convert_to_double() { - let datum = Datum::float(2.5); + let datum = Datum::float(2.5_f32); let result = datum.to(&Primitive(PrimitiveType::Double)).unwrap(); @@ -1358,10 +1358,10 @@ fn test_iceberg_float_order() { Datum::float(f32::MIN), Datum::float(f32::INFINITY), Datum::float(-f32::INFINITY), - Datum::float(1.0), - Datum::float(-1.0), - Datum::float(0.0), - Datum::float(-0.0), + Datum::float(1.0_f32), + Datum::float(-1.0_f32), + Datum::float(0.0_f32), + Datum::float(-0.0_f32), ]; let mut float_sorted = float_values.clone(); @@ -1371,10 +1371,10 @@ fn test_iceberg_float_order() { Datum::float(-f32::NAN), Datum::float(-f32::INFINITY), Datum::float(f32::MIN), - Datum::float(-1.0), - Datum::float(-0.0), - Datum::float(0.0), - Datum::float(1.0), + Datum::float(-1.0_f32), + Datum::float(-0.0_f32), + Datum::float(0.0_f32), + Datum::float(1.0_f32), Datum::float(f32::MAX), Datum::float(f32::INFINITY), Datum::float(f32::NAN), @@ -1418,8 +1418,8 @@ fn test_iceberg_float_order() { #[test] fn test_negative_zero_less_than_positive_zero() { { - let neg_zero = Datum::float(-0.0); - let pos_zero = Datum::float(0.0); + let neg_zero = Datum::float(-0.0_f32); + let pos_zero = Datum::float(0.0_f32); assert_eq!( neg_zero.partial_cmp(&pos_zero), diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 95654c89a7..2d9ef2077c 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -1489,7 +1489,7 @@ mod tests { (0, Datum::bool(false)), (1, Datum::int(1)), (2, Datum::long(1)), - (3, Datum::float(0.5)), + (3, Datum::float(0.5_f32)), (4, Datum::double(0.5)), (5, Datum::string("a")), (6, Datum::binary(vec![])), @@ -1529,7 +1529,7 @@ mod tests { (0, Datum::bool(true)), (1, Datum::int(4)), (2, Datum::long(4)), - (3, Datum::float(3.5)), + (3, Datum::float(3.5_f32)), (4, Datum::double(3.5)), (5, Datum::string("d")), (6, Datum::binary(vec![122, 122, 122, 122])), @@ -1938,11 +1938,11 @@ mod tests { assert_eq!(*data_file.value_counts(), HashMap::from([(0, 4), (1, 4)])); assert_eq!( *data_file.lower_bounds(), - HashMap::from([(0, Datum::float(1.0)), (1, Datum::double(1.0)),]) + HashMap::from([(0, Datum::float(1.0_f32)), (1, Datum::double(1.0)),]) ); assert_eq!( *data_file.upper_bounds(), - HashMap::from([(0, Datum::float(2.0)), (1, Datum::double(2.0)),]) + HashMap::from([(0, Datum::float(2.0_f32)), (1, Datum::double(2.0)),]) ); assert_eq!( *data_file.null_value_counts(), @@ -2078,11 +2078,11 @@ mod tests { assert_eq!(*data_file.value_counts(), HashMap::from([(4, 4), (7, 4)])); assert_eq!( *data_file.lower_bounds(), - HashMap::from([(4, Datum::float(1.0)), (7, Datum::float(1.0)),]) + HashMap::from([(4, Datum::float(1.0_f32)), (7, Datum::float(1.0_f32)),]) ); assert_eq!( *data_file.upper_bounds(), - HashMap::from([(4, Datum::float(2.0)), (7, Datum::float(2.0)),]) + HashMap::from([(4, Datum::float(2.0_f32)), (7, Datum::float(2.0_f32)),]) ); assert_eq!( *data_file.null_value_counts(), @@ -2242,11 +2242,11 @@ mod tests { assert_eq!(*data_file.value_counts(), HashMap::from([(1, 4), (4, 4)])); assert_eq!( *data_file.lower_bounds(), - HashMap::from([(1, Datum::float(1.0)), (4, Datum::float(1.0))]) + HashMap::from([(1, Datum::float(1.0_f32)), (4, Datum::float(1.0_f32))]) ); assert_eq!( *data_file.upper_bounds(), - HashMap::from([(1, Datum::float(2.0)), (4, Datum::float(2.0))]) + HashMap::from([(1, Datum::float(2.0_f32)), (4, Datum::float(2.0_f32))]) ); assert_eq!( *data_file.null_value_counts(), @@ -2428,18 +2428,18 @@ mod tests { *data_file.lower_bounds(), HashMap::from([ (1, Datum::int(1)), - (2, Datum::float(1.0)), + (2, Datum::float(1.0_f32)), (6, Datum::int(1)), - (7, Datum::float(1.0)) + (7, Datum::float(1.0_f32)) ]) ); assert_eq!( *data_file.upper_bounds(), HashMap::from([ (1, Datum::int(4)), - (2, Datum::float(2.0)), + (2, Datum::float(2.0_f32)), (6, Datum::int(4)), - (7, Datum::float(2.0)) + (7, Datum::float(2.0_f32)) ]) ); assert_eq!( diff --git a/crates/sqllogictest/src/schedule.rs b/crates/sqllogictest/src/schedule.rs index 25728a2968..29e37c5c72 100644 --- a/crates/sqllogictest/src/schedule.rs +++ b/crates/sqllogictest/src/schedule.rs @@ -110,7 +110,7 @@ impl Schedule { let step_sql_path = PathBuf::from(format!( "{}/testdata/slts/{}", env!("CARGO_MANIFEST_DIR"), - &step.slt + step.slt )); engine.run_slt_file(&step_sql_path).await?; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 217e2256bd..359cdaf68d 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -20,5 +20,5 @@ # # The channel is exactly same day for our MSRV. [toolchain] -channel = "nightly-2026-03-05" +channel = "nightly-2026-04-16" components = ["rustfmt", "clippy"] From 4200aa0a7bc0b9f23ff7b2a6abd626af673d59d3 Mon Sep 17 00:00:00 2001 From: Daniel Carl Jones Date: Fri, 28 Aug 2026 07:23:18 +0100 Subject: [PATCH 03/74] chore(repo): Replace bug report version selector with freeform text field (#2826) * chore(repo): Update Apache Iceberg Rust version options Updated the version options for Apache Iceberg Rust in the bug report template. * Replace optional version list with required freeform text --- .github/ISSUE_TEMPLATE/iceberg_bug_report.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml b/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml index 0759a65723..9f938b11a1 100644 --- a/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml @@ -22,19 +22,15 @@ description: Problems, bugs and issues with Apache Iceberg Rust labels: bug body: - - type: dropdown + - type: input attributes: label: Apache Iceberg Rust version - description: What Apache Iceberg Rust version are you using? - multiple: false - options: - - 0.6.0 (latest version) - - 0.5.1 - - 0.4.0 - - 0.3.0 - - 0.2.0 + description: > + What Apache Iceberg Rust version are you using? + placeholder: > + For example: v0.10.0 validations: - required: false + required: true - type: textarea attributes: label: Describe the bug From 6678583b4c07c2812107d1aee239864a4cf515b0 Mon Sep 17 00:00:00 2001 From: Matt Faltyn <76570855+mattfaltyn@users.noreply.github.com> Date: Fri, 28 Aug 2026 08:24:22 +0200 Subject: [PATCH 04/74] fix(sql): propagate catalog commit errors (#3062) * fix(sql): propagate catalog commit errors * test(sql): clarify transaction commit coverage * test(sql): exercise commit errors through catalog API * test(sql): keep commit regression catalog-focused --- crates/catalog/sql/src/catalog.rs | 92 +++++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/crates/catalog/sql/src/catalog.rs b/crates/catalog/sql/src/catalog.rs index 76cfc11bc8..73671f5c4b 100644 --- a/crates/catalog/sql/src/catalog.rs +++ b/crates/catalog/sql/src/catalog.rs @@ -564,9 +564,12 @@ impl SqlCatalog { Some(t) => sqlx_query.execute(&mut **t).await.map_err(from_sqlx_error), None => { let mut tx = self.connection.begin().await.map_err(from_sqlx_error)?; - let result = sqlx_query.execute(&mut *tx).await.map_err(from_sqlx_error); - let _ = tx.commit().await.map_err(from_sqlx_error); - result + let result = sqlx_query + .execute(&mut *tx) + .await + .map_err(from_sqlx_error)?; + tx.commit().await.map_err(from_sqlx_error)?; + Ok(result) } } } @@ -1242,10 +1245,10 @@ mod tests { use crate::catalog::{ CATALOG_FIELD_RECORD_TYPE, CATALOG_TABLE_NAME, NAMESPACE_LOCATION_PROPERTY_KEY, - SQL_CATALOG_PROP_BIND_STYLE, SQL_CATALOG_PROP_BIND_STYLE_LEGACY, + NAMESPACE_TABLE_NAME, SQL_CATALOG_PROP_BIND_STYLE, SQL_CATALOG_PROP_BIND_STYLE_LEGACY, SQL_CATALOG_PROP_SCHEMA_VERSION, SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, }; - use crate::{SchemaVersion, SqlBindStyle, SqlCatalogBuilder}; + use crate::{SchemaVersion, SqlBindStyle, SqlCatalog, SqlCatalogBuilder}; const UUID_REGEX_STR: &str = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; @@ -1386,6 +1389,85 @@ mod tests { new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; } + async fn new_commit_error_catalog() -> SqlCatalog { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .prop("pool.max-connections", "1") + .load( + "iceberg", + HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri), + (SQL_CATALOG_PROP_WAREHOUSE.to_string(), temp_path()), + ]), + ) + .await + .unwrap(); + + catalog + .connection + .execute("PRAGMA foreign_keys = ON") + .await + .unwrap(); + // This deferred constraint lets an INSERT succeed while COMMIT fails. + catalog + .connection + .execute("CREATE TABLE parent(id INTEGER PRIMARY KEY)") + .await + .unwrap(); + catalog + .connection + .execute( + "CREATE TABLE child(parent_id INTEGER REFERENCES parent(id) \ + DEFERRABLE INITIALLY DEFERRED)", + ) + .await + .unwrap(); + + catalog + } + + #[tokio::test] + async fn test_execute_returns_commit_error() { + let catalog = new_commit_error_catalog().await; + + // Make the public namespace operation insert a child row whose deferred + // foreign-key constraint succeeds during execution but fails at commit. + let trigger = format!( + "CREATE TRIGGER fail_namespace_commit + AFTER INSERT ON {NAMESPACE_TABLE_NAME} + BEGIN INSERT INTO child VALUES (1); END" + ); + catalog.connection.execute(trigger.as_str()).await.unwrap(); + + let failed_namespace = NamespaceIdent::new("failed".into()); + let error = catalog + .create_namespace(&failed_namespace, HashMap::new()) + .await + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(!catalog.namespace_exists(&failed_namespace).await.unwrap()); + + // A valid relationship confirms that successful transactions still commit. + catalog + .connection + .execute("INSERT INTO parent VALUES (1)") + .await + .unwrap(); + let committed_namespace = NamespaceIdent::new("committed".into()); + catalog + .create_namespace(&committed_namespace, HashMap::new()) + .await + .unwrap(); + assert!( + catalog + .namespace_exists(&committed_namespace) + .await + .unwrap() + ); + } + // Regression test: storage-backend props set on the catalog must reach // the FileIO; otherwise authenticated backends fail with 401s on writes. #[tokio::test] From 76572cf58ef255201b95816e5f8178a7b351b8d8 Mon Sep 17 00:00:00 2001 From: Joseph Lenton Date: Fri, 28 Aug 2026 07:42:57 +0100 Subject: [PATCH 05/74] refactor: change from serialising a u64 to i64 (#3065) --- crates/iceberg/src/spec/manifest_list/_serde.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/spec/manifest_list/_serde.rs b/crates/iceberg/src/spec/manifest_list/_serde.rs index 1756d5bbfa..2a10c39ea3 100644 --- a/crates/iceberg/src/spec/manifest_list/_serde.rs +++ b/crates/iceberg/src/spec/manifest_list/_serde.rs @@ -193,7 +193,7 @@ pub(super) struct ManifestFileV3 { pub deleted_rows_count: i64, pub partitions: Option>, pub key_metadata: Option, - pub first_row_id: Option, + pub first_row_id: Option, } impl ManifestFileV3 { @@ -215,7 +215,7 @@ impl ManifestFileV3 { deleted_rows_count: Some(self.deleted_rows_count.try_into()?), partitions: self.partitions, key_metadata: self.key_metadata.map(|b| b.into_vec()), - first_row_id: self.first_row_id, + first_row_id: self.first_row_id.map(|n| n.try_into()).transpose()?, }; Ok(manifest_file) @@ -372,7 +372,7 @@ impl TryFrom for ManifestFileV3 { .try_into()?, partitions: value.partitions, key_metadata, - first_row_id: value.first_row_id, + first_row_id: value.first_row_id.map(|n| n.try_into()).transpose()?, }) } } From 57a5612e0ae5327c04daa2b2b616849822aa3e6d Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:29:17 +0800 Subject: [PATCH 06/74] feat: add lazy property view macro (#3044) --- crates/property-macro/README.md | 92 +++++++- crates/property-macro/public-api.txt | 1 + crates/property-macro/src/lib.rs | 14 +- crates/property-macro/src/properties.rs | 62 +++-- crates/property-macro/src/properties_view.rs | 212 ++++++++++++++++++ ...dditional_keys_without_parse_properties.rs | 0 ...ional_keys_without_parse_properties.stderr | 2 +- .../missing_property.rs | 0 .../missing_property.stderr | 2 +- .../multiple_sources.rs | 0 .../multiple_sources.stderr | 2 +- .../{ => derive-properties}/nested_default.rs | 0 .../nested_default.stderr | 2 +- ...arse_properties_without_additional_keys.rs | 0 ..._properties_without_additional_keys.stderr | 2 +- .../{ => derive-properties}/prefix_default.rs | 0 .../prefix_default.stderr | 2 +- .../{ => derive-properties}/prefix_non_map.rs | 0 .../prefix_non_map.stderr | 2 +- ...dditional_keys_without_parse_properties.rs | 27 +++ ...ional_keys_without_parse_properties.stderr | 6 + .../view_generic_declaration.rs | 27 +++ .../view_generic_declaration.stderr | 5 + .../properties-view/view_missing_property.rs | 26 +++ .../view_missing_property.stderr | 5 + .../properties-view/view_multiple_sources.rs | 29 +++ .../view_multiple_sources.stderr | 14 ++ .../properties-view/view_nested_default.rs | 34 +++ .../view_nested_default.stderr | 6 + ...arse_properties_without_additional_keys.rs | 38 ++++ ..._properties_without_additional_keys.stderr | 6 + .../properties-view/view_prefix_default.rs | 29 +++ .../view_prefix_default.stderr | 14 ++ .../properties-view/view_prefix_non_map.rs | 27 +++ .../view_prefix_non_map.stderr | 5 + .../properties-view/view_tuple_declaration.rs | 24 ++ .../view_tuple_declaration.stderr | 5 + crates/property-macro/tests/compile_fail.rs | 9 +- .../{properties.rs => properties_derive.rs} | 0 .../property-macro/tests/properties_view.rs | 200 +++++++++++++++++ 40 files changed, 890 insertions(+), 41 deletions(-) create mode 100644 crates/property-macro/src/properties_view.rs rename crates/property-macro/tests/compile-fail/{ => derive-properties}/additional_keys_without_parse_properties.rs (100%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/additional_keys_without_parse_properties.stderr (68%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/missing_property.rs (100%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/missing_property.stderr (59%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/multiple_sources.rs (100%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/multiple_sources.stderr (79%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/nested_default.rs (100%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/nested_default.stderr (77%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/parse_properties_without_additional_keys.rs (100%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/parse_properties_without_additional_keys.stderr (69%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/prefix_default.rs (100%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/prefix_default.stderr (79%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/prefix_non_map.rs (100%) rename crates/property-macro/tests/compile-fail/{ => derive-properties}/prefix_non_map.stderr (62%) create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.stderr create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.stderr create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_missing_property.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_missing_property.stderr create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.stderr create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_nested_default.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_nested_default.stderr create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.stderr create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.stderr create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.stderr create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.rs create mode 100644 crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.stderr rename crates/property-macro/tests/{properties.rs => properties_derive.rs} (100%) create mode 100644 crates/property-macro/tests/properties_view.rs diff --git a/crates/property-macro/README.md b/crates/property-macro/README.md index 8ebf7054c7..ea746a4973 100644 --- a/crates/property-macro/README.md +++ b/crates/property-macro/README.md @@ -17,14 +17,16 @@ under the License. --> -# Iceberg property derive macro +# Iceberg property macros -`Properties` parses a typed struct from a flat `HashMap` and -can generate opt-in read-only getters. It deliberately does not generate -property-map serialization or implement `Default`, `Serialize`, `Deserialize`, -or any other trait. +## `Properties` derive macro -## Generated API +`#[derive(Properties)]` parses an owned typed struct from a flat +`HashMap` and can generate opt-in read-only getters. It +deliberately does not generate property-map serialization or implement +`Default`, `Serialize`, `Deserialize`, or any other trait. + +### Generated API For every annotated struct, `#[derive(Properties)]` generates this inherent constructor: @@ -54,7 +56,7 @@ map. corresponding types from `std`. The macro recognizes those field shapes syntactically and generates code using the standard-library variants. -## Complete example +### Complete example This example covers exact keys and defaults, optional values, case-insensitive booleans, prefixed maps, nested groups, custom single-value parsing, custom @@ -211,7 +213,7 @@ fn main() -> iceberg::Result<()> { } ``` -## Using ordinary derives together +### Using ordinary derives together `Properties` does not implicitly derive other traits, so `Default`, `Serialize`, and `Deserialize` can be selected independently and behave like @@ -251,6 +253,8 @@ fn main() -> Result<(), Box> { } ``` +### Property declarations + All field settings must be grouped under `#[property(...)]`. This keeps `key`, `default`, `prefix`, `nested`, parser hooks, additional keys, and getter generation in one attribute and avoids collisions with ordinary Rust derives. @@ -271,3 +275,75 @@ converted into their field type with `Into`. `from_properties` and both custom parser hooks use `iceberg::Result`. Generated `FromStr` failures use `ErrorKind::DataInvalid`. The macro preserves errors from custom parsers and adds the primary property key as error context. + +## `properties_view!` function-like macro + +`properties_view!` independently defines a lightweight borrowed view over a +flat property map. Its struct-shaped fields are declarations used to generate +getters rather than stored fields. Constructing the view does not parse +anything; each generated getter parses and returns only its declared property. +Missing properties use their annotated defaults, while invalid configured +values return an error from the corresponding getter. + +```rust +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +properties_view! { + #[derive(Debug)] + pub struct WriteProperties { + // `getter` makes a generated getter public. Without it, the getter is + // private, regardless of the declaration's visibility modifier. + + /// Number of times to retry a commit. + #[property(key = "commit.retry.num-retries", default = 4, getter)] + commit_num_retries: usize, + + /// Optional configured base directory for metadata files. + #[property(key = "write.metadata.path", default = None)] + raw_write_metadata_path: Option, + } +} + +impl WriteProperties<'_> { + /// Returns the configured metadata path or the table's metadata directory. + pub fn write_metadata_path(&self, table_path: &str) -> iceberg::Result { + Ok(self.raw_write_metadata_path()?.unwrap_or_else(|| { + format!("{}/metadata", table_path.trim_end_matches('/')) + })) + } +} + +fn read(properties: &HashMap) -> iceberg::Result<()> { + let view = WriteProperties::new(properties); + let _retries = view.commit_num_retries()?; + let _metadata_path = view.write_metadata_path("s3://bucket/table")?; + Ok(()) +} +``` + +For this declaration, the macro generates +`impl<'properties> WriteProperties<'properties>` with a +`pub fn new(properties: &'properties HashMap) -> Self` +constructor. The constructor has the same visibility as the view declaration: +`pub` here, `pub(crate)` for a `pub(crate)` view, and private for a private view. +It only stores the map reference and does not parse any properties. + +The generated type has one lifetime parameter and contains only a reference to +the source `HashMap`; no declared field is stored in the view. Every declaration +generates a getter, but only the `getter` option makes that method public. Field +visibility modifiers are ignored, consistent with how `Properties` uses +`getter` to control generated accessors. Omitting `getter` makes it possible to +keep a raw generated method private and expose a public method that adds +application-specific behavior, as shown by `write_metadata_path` above. + +Non-`Copy` values are returned by value because the view does not retain parsed +results, so calling the same getter repeatedly reparses that field. + +The view supports exact keys, optional values, case-insensitive booleans, +prefixed maps, nested views, `parse_with`, and `parse_properties_with` using the +same property declaration syntax as `Properties`. A nested field's type must be +another view type, such as `CommitProperties<'_>`. The nested getter returns a +view tied directly to the source map's lifetime, so it remains usable after the +parent view is dropped. diff --git a/crates/property-macro/public-api.txt b/crates/property-macro/public-api.txt index 8edb5d0952..23e6c068d0 100644 --- a/crates/property-macro/public-api.txt +++ b/crates/property-macro/public-api.txt @@ -1,2 +1,3 @@ pub mod iceberg_property_macro pub proc macro iceberg_property_macro::#[derive(Properties)] +pub proc macro iceberg_property_macro::properties_view!() diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs index e73b48c0f7..e2e3d871da 100644 --- a/crates/property-macro/src/lib.rs +++ b/crates/property-macro/src/lib.rs @@ -18,9 +18,10 @@ #![doc = include_str!("../README.md")] use proc_macro::TokenStream; -use syn::{DeriveInput, parse_macro_input}; +use syn::{DeriveInput, ItemStruct, parse_macro_input}; mod properties; +mod properties_view; /// Derives property-map parsing and opt-in read-only accessors for a struct. #[proc_macro_derive(Properties, attributes(property))] @@ -32,3 +33,14 @@ pub fn derive_properties(input: TokenStream) -> TokenStream { Err(error) => error.into_compile_error().into(), } } + +/// Defines a borrowed view that parses properties independently through generated getters. +#[proc_macro] +pub fn properties_view(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as ItemStruct); + + match properties_view::expand_properties_view(input) { + Ok(tokens) => tokens.into(), + Err(error) => error.into_compile_error().into(), + } +} diff --git a/crates/property-macro/src/properties.rs b/crates/property-macro/src/properties.rs index b4fcf2d63e..abedd9de56 100644 --- a/crates/property-macro/src/properties.rs +++ b/crates/property-macro/src/properties.rs @@ -24,20 +24,20 @@ use syn::{ Ident, Lit, Path, PathArguments, Token, Type, }; -struct PropertyField { - ident: Ident, - ty: Type, +pub(crate) struct PropertyField { + pub(crate) ident: Ident, + pub(crate) ty: Type, key: Option, additional_keys: Option>, prefix: Option, - nested: bool, + pub(crate) nested: bool, default: Option, parse_with: Option, parse_properties_with: Option, option_inner_type: Option, map_value_type: Option, - public_getter: bool, - doc_attributes: Vec, + pub(crate) public_getter: bool, + pub(crate) doc_attributes: Vec, } enum PropertyOption { @@ -52,7 +52,7 @@ enum PropertyOption { } #[derive(Default)] -struct PropertyOptions { +pub(crate) struct PropertyOptions { key: Option, additional_keys: Option>, prefix: Option, @@ -143,7 +143,7 @@ pub(crate) fn expand_properties(input: DeriveInput) -> syn::Result }) } -fn parse_property_field( +pub(crate) fn parse_property_field( field: &Field, property_options: PropertyOptions, ) -> syn::Result { @@ -244,7 +244,7 @@ fn parse_property_field( }) } -fn property_options(field: &Field) -> syn::Result { +pub(crate) fn property_options(field: &Field) -> syn::Result { let Some(attribute) = find_attribute(&field.attrs, "property")? else { return Err(Error::new_spanned( field, @@ -394,9 +394,27 @@ fn find_attribute<'a>( fn parse_field(field: &PropertyField) -> syn::Result { let ident = &field.ident; + let parse = parse_field_value(field, ParseTarget::Owned)?; + Ok(quote!(#ident: #parse)) +} + +#[derive(Clone, Copy)] +pub(crate) enum ParseTarget { + Owned, + View, +} + +pub(crate) fn parse_field_value( + field: &PropertyField, + target: ParseTarget, +) -> syn::Result { if field.nested { let ty = &field.ty; - return Ok(quote!(#ident: <#ty>::from_properties(properties)?)); + let constructor = match target { + ParseTarget::Owned => quote!(<#ty>::from_properties(properties)?), + ParseTarget::View => quote!(<#ty>::new(properties)), + }; + return Ok(constructor); } let ty = &field.ty; @@ -415,17 +433,15 @@ fn parse_field(field: &PropertyField) -> syn::Result { ) })?; let default = typed_default(field)?; - return Ok(quote! { - #ident: { - let parsed: ::iceberg::Result<#ty> = #parse_properties_with( - properties, - #key, - &[#(#additional_keys),*], - #default, - ); - parsed.map_err(|error| error.with_context("property", #key))? - } - }); + return Ok(quote! {{ + let parsed: ::iceberg::Result<#ty> = #parse_properties_with( + properties, + #key, + &[#(#additional_keys),*], + #default, + ); + parsed.map_err(|error| error.with_context("property", #key))? + }}); } if let Some(prefix) = &field.prefix { @@ -441,7 +457,7 @@ fn parse_field(field: &PropertyField) -> syn::Result { quote!(value.parse::<#value_type>()) }; return Ok(quote! { - #ident: properties + properties .iter() .filter_map(|(key, value)| { key.strip_prefix(#prefix).map(|suffix| { @@ -512,7 +528,7 @@ fn parse_field(field: &PropertyField) -> syn::Result { }; Ok(quote! { - #ident: match properties.get(#key) { + match properties.get(#key) { Some(value) => #parse, None => #default, } diff --git a/crates/property-macro/src/properties_view.rs b/crates/property-macro/src/properties_view.rs new file mode 100644 index 0000000000..7536f57636 --- /dev/null +++ b/crates/property-macro/src/properties_view.rs @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{Error, Fields, GenericArgument, ItemStruct, Lifetime, PathArguments, Type}; + +use crate::properties::{ + ParseTarget, PropertyField, parse_field_value, parse_property_field, property_options, +}; + +pub(crate) fn expand_properties_view(input: ItemStruct) -> syn::Result { + if !input.generics.params.is_empty() || input.generics.where_clause.is_some() { + return Err(Error::new_spanned( + input.generics, + "properties_view! does not support generic declarations", + )); + } + + let attributes = input.attrs; + let visibility = input.vis; + let struct_name = input.ident; + let fields = match input.fields { + Fields::Named(fields) => fields.named, + _ => { + return Err(Error::new_spanned( + struct_name, + "properties_view! requires a struct-shaped declaration with named fields", + )); + } + }; + let fields = fields + .iter() + .map(|field| parse_property_field(field, property_options(field)?)) + .collect::>>()?; + let properties_lifetime: Lifetime = syn::parse_quote!('properties); + let getters = fields + .iter() + .map(|field| view_field_getter(field, &properties_lifetime)) + .collect::>>()?; + + Ok(quote! { + #(#attributes)* + #visibility struct #struct_name<#properties_lifetime> { + properties: &#properties_lifetime ::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + } + + impl<#properties_lifetime> #struct_name<#properties_lifetime> { + /// Creates a property view without parsing any values. + #visibility fn new( + properties: &#properties_lifetime ::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + ) -> Self { + Self { properties } + } + + #(#getters)* + } + }) +} + +fn view_field_getter( + field: &PropertyField, + properties_lifetime: &Lifetime, +) -> syn::Result { + let ident = &field.ident; + let ty = view_field_type(field, properties_lifetime)?; + let docs = &field.doc_attributes; + let visibility = if field.public_getter { + quote!(pub) + } else { + TokenStream2::new() + }; + let parse = parse_field_value(field, ParseTarget::View)?; + + Ok(quote! { + #(#docs)* + #visibility fn #ident(&self) -> ::iceberg::Result<#ty> { + let properties = self.properties; + let value = #parse; + Ok(value) + } + }) +} + +fn view_field_type(field: &PropertyField, properties_lifetime: &Lifetime) -> syn::Result { + let mut ty = field.ty.clone(); + if !field.nested { + return Ok(ty); + } + + let Type::Path(type_path) = &mut ty else { + return Err(Error::new_spanned( + &field.ty, + "nested property view types must have a lifetime argument", + )); + }; + let Some(segment) = type_path.path.segments.last_mut() else { + return Err(Error::new_spanned( + &field.ty, + "nested property view types must have a lifetime argument", + )); + }; + let PathArguments::AngleBracketed(arguments) = &mut segment.arguments else { + return Err(Error::new_spanned( + &field.ty, + "nested property view types must have a lifetime argument", + )); + }; + let Some(lifetime) = arguments + .args + .iter_mut() + .find_map(|argument| match argument { + GenericArgument::Lifetime(lifetime) => Some(lifetime), + _ => None, + }) + else { + return Err(Error::new_spanned( + &field.ty, + "nested property view types must have a lifetime argument", + )); + }; + *lifetime = properties_lifetime.clone(); + Ok(ty) +} + +#[cfg(test)] +mod tests { + use syn::{ImplItem, Item, Visibility, parse_quote, parse2}; + + use super::*; + + fn method_visibility(input: ItemStruct, method_name: &str) -> Visibility { + let expanded = expand_properties_view(input).unwrap(); + let file = parse2::(expanded).unwrap(); + + file.items + .into_iter() + .find_map(|item| match item { + Item::Impl(item_impl) => item_impl.items.into_iter().find_map(|item| match item { + ImplItem::Fn(method) if method.sig.ident == method_name => Some(method.vis), + _ => None, + }), + _ => None, + }) + .unwrap() + } + + #[test] + fn constructor_uses_the_struct_visibility() { + let visibility = method_visibility( + parse_quote! { + pub(crate) struct TestProperties { + #[property(key = "retries", default = 4)] + retries: u64, + } + }, + "new", + ); + + assert!(matches!(visibility, Visibility::Restricted(_))); + } + + #[test] + fn getter_option_overrides_explicit_visibility() { + let visibility = method_visibility( + parse_quote! { + struct TestProperties { + #[property(key = "retries", default = 4, getter)] + pub(crate) retries: u64, + } + }, + "retries", + ); + + assert!(matches!(visibility, Visibility::Public(_))); + } + + #[test] + fn field_visibility_does_not_make_the_getter_public() { + let visibility = method_visibility( + parse_quote! { + struct TestProperties { + #[property(key = "retries", default = 4)] + pub retries: u64, + } + }, + "retries", + ); + + assert!(matches!(visibility, Visibility::Inherited)); + } +} diff --git a/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.rs b/crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.rs similarity index 100% rename from crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.rs rename to crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.rs diff --git a/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.stderr b/crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.stderr similarity index 68% rename from crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.stderr rename to crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.stderr index 905ffb0043..b0fa9be55c 100644 --- a/crates/property-macro/tests/compile-fail/additional_keys_without_parse_properties.stderr +++ b/crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.stderr @@ -1,5 +1,5 @@ error: additional_keys requires parse_properties_with in #[property(...)] - --> tests/compile-fail/additional_keys_without_parse_properties.rs:23:5 + --> tests/compile-fail/derive-properties/additional_keys_without_parse_properties.rs:23:5 | 23 | / #[property(key = "value", additional_keys = ["other"], default = 1)] 24 | | value: u64, diff --git a/crates/property-macro/tests/compile-fail/missing_property.rs b/crates/property-macro/tests/compile-fail/derive-properties/missing_property.rs similarity index 100% rename from crates/property-macro/tests/compile-fail/missing_property.rs rename to crates/property-macro/tests/compile-fail/derive-properties/missing_property.rs diff --git a/crates/property-macro/tests/compile-fail/missing_property.stderr b/crates/property-macro/tests/compile-fail/derive-properties/missing_property.stderr similarity index 59% rename from crates/property-macro/tests/compile-fail/missing_property.stderr rename to crates/property-macro/tests/compile-fail/derive-properties/missing_property.stderr index 3555f484d6..5be8dfe96c 100644 --- a/crates/property-macro/tests/compile-fail/missing_property.stderr +++ b/crates/property-macro/tests/compile-fail/derive-properties/missing_property.stderr @@ -1,5 +1,5 @@ error: Properties fields must declare #[property(...)] - --> tests/compile-fail/missing_property.rs:23:5 + --> tests/compile-fail/derive-properties/missing_property.rs:23:5 | 23 | value: u64, | ^^^^^^^^^^ diff --git a/crates/property-macro/tests/compile-fail/multiple_sources.rs b/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.rs similarity index 100% rename from crates/property-macro/tests/compile-fail/multiple_sources.rs rename to crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.rs diff --git a/crates/property-macro/tests/compile-fail/multiple_sources.stderr b/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.stderr similarity index 79% rename from crates/property-macro/tests/compile-fail/multiple_sources.stderr rename to crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.stderr index 2e4c67f868..37dc87f5e1 100644 --- a/crates/property-macro/tests/compile-fail/multiple_sources.stderr +++ b/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.stderr @@ -1,5 +1,5 @@ error: Properties fields must declare exactly one of key, prefix, or nested in #[property(...)] - --> tests/compile-fail/multiple_sources.rs:25:5 + --> tests/compile-fail/derive-properties/multiple_sources.rs:25:5 | 25 | / #[property(key = "value", prefix = "values.", default = HashMap::new())] 26 | | value: HashMap, diff --git a/crates/property-macro/tests/compile-fail/nested_default.rs b/crates/property-macro/tests/compile-fail/derive-properties/nested_default.rs similarity index 100% rename from crates/property-macro/tests/compile-fail/nested_default.rs rename to crates/property-macro/tests/compile-fail/derive-properties/nested_default.rs diff --git a/crates/property-macro/tests/compile-fail/nested_default.stderr b/crates/property-macro/tests/compile-fail/derive-properties/nested_default.stderr similarity index 77% rename from crates/property-macro/tests/compile-fail/nested_default.stderr rename to crates/property-macro/tests/compile-fail/derive-properties/nested_default.stderr index 6db683674a..878985957a 100644 --- a/crates/property-macro/tests/compile-fail/nested_default.stderr +++ b/crates/property-macro/tests/compile-fail/derive-properties/nested_default.stderr @@ -1,5 +1,5 @@ error: nested fields obtain defaults from their own property annotations and cannot declare default in #[property(...)] - --> tests/compile-fail/nested_default.rs:29:5 + --> tests/compile-fail/derive-properties/nested_default.rs:29:5 | 29 | / #[property(nested, default = unreachable!())] 30 | | inner: Inner, diff --git a/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.rs b/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.rs similarity index 100% rename from crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.rs rename to crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.rs diff --git a/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.stderr b/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.stderr similarity index 69% rename from crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.stderr rename to crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.stderr index ff2168f92f..e4eae1fea6 100644 --- a/crates/property-macro/tests/compile-fail/parse_properties_without_additional_keys.stderr +++ b/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.stderr @@ -1,5 +1,5 @@ error: parse_properties_with requires additional_keys in #[property(...)] - --> tests/compile-fail/parse_properties_without_additional_keys.rs:34:5 + --> tests/compile-fail/derive-properties/parse_properties_without_additional_keys.rs:34:5 | 34 | / #[property(key = "value", default = 1, parse_properties_with = parse_all)] 35 | | value: u64, diff --git a/crates/property-macro/tests/compile-fail/prefix_default.rs b/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.rs similarity index 100% rename from crates/property-macro/tests/compile-fail/prefix_default.rs rename to crates/property-macro/tests/compile-fail/derive-properties/prefix_default.rs diff --git a/crates/property-macro/tests/compile-fail/prefix_default.stderr b/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.stderr similarity index 79% rename from crates/property-macro/tests/compile-fail/prefix_default.stderr rename to crates/property-macro/tests/compile-fail/derive-properties/prefix_default.stderr index 9dcddded2d..902e95782b 100644 --- a/crates/property-macro/tests/compile-fail/prefix_default.stderr +++ b/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.stderr @@ -1,5 +1,5 @@ error: prefix fields collect matching properties and cannot declare default in #[property(...)] - --> tests/compile-fail/prefix_default.rs:25:5 + --> tests/compile-fail/derive-properties/prefix_default.rs:25:5 | 25 | / #[property(prefix = "values.", default = HashMap::new())] 26 | | values: HashMap, diff --git a/crates/property-macro/tests/compile-fail/prefix_non_map.rs b/crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.rs similarity index 100% rename from crates/property-macro/tests/compile-fail/prefix_non_map.rs rename to crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.rs diff --git a/crates/property-macro/tests/compile-fail/prefix_non_map.stderr b/crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.stderr similarity index 62% rename from crates/property-macro/tests/compile-fail/prefix_non_map.stderr rename to crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.stderr index 72627bc592..41454f966b 100644 --- a/crates/property-macro/tests/compile-fail/prefix_non_map.stderr +++ b/crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.stderr @@ -1,5 +1,5 @@ error: property prefix fields must have type HashMap - --> tests/compile-fail/prefix_non_map.rs:24:12 + --> tests/compile-fail/derive-properties/prefix_non_map.rs:24:12 | 24 | value: u64, | ^^^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs b/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs new file mode 100644 index 0000000000..71aa916ffc --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct AdditionalKeysWithoutParser { + #[property(key = "value", additional_keys = ["other"], default = 1)] + value: u64, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.stderr new file mode 100644 index 0000000000..a5c74eee0a --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.stderr @@ -0,0 +1,6 @@ +error: additional_keys requires parse_properties_with in #[property(...)] + --> tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs:22:9 + | +22 | / #[property(key = "value", additional_keys = ["other"], default = 1)] +23 | | value: u64, + | |__________________^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.rs b/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.rs new file mode 100644 index 0000000000..af83d0fa82 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct GenericView { + #[property(key = "value", default = None)] + value: Option, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.stderr new file mode 100644 index 0000000000..2dc8658cd7 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.stderr @@ -0,0 +1,5 @@ +error: properties_view! does not support generic declarations + --> tests/compile-fail/properties-view/view_generic_declaration.rs:21:23 + | +21 | struct GenericView { + | ^^^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.rs b/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.rs new file mode 100644 index 0000000000..c0d7f84f5c --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.rs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct MissingProperty { + value: u64, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.stderr new file mode 100644 index 0000000000..8f0a522acd --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.stderr @@ -0,0 +1,5 @@ +error: Properties fields must declare #[property(...)] + --> tests/compile-fail/properties-view/view_missing_property.rs:22:9 + | +22 | value: u64, + | ^^^^^^^^^^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.rs b/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.rs new file mode 100644 index 0000000000..db8d68bcce --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +properties_view! { + struct MultipleSources { + #[property(key = "value", prefix = "values.", default = HashMap::new())] + value: HashMap, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.stderr new file mode 100644 index 0000000000..83cc348cbc --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.stderr @@ -0,0 +1,14 @@ +error: Properties fields must declare exactly one of key, prefix, or nested in #[property(...)] + --> tests/compile-fail/properties-view/view_multiple_sources.rs:24:9 + | +24 | / #[property(key = "value", prefix = "values.", default = HashMap::new())] +25 | | value: HashMap, + | |___________________________________^ + +warning: unused import: `std::collections::HashMap` + --> tests/compile-fail/properties-view/view_multiple_sources.rs:18:5 + | +18 | use std::collections::HashMap; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.rs b/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.rs new file mode 100644 index 0000000000..06ef741f2c --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct Inner { + #[property(key = "value", default = 1)] + value: u64, + } +} + +properties_view! { + struct NestedDefault { + #[property(nested, default = unreachable!())] + inner: Inner<'_>, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.stderr new file mode 100644 index 0000000000..2c024f1e61 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.stderr @@ -0,0 +1,6 @@ +error: nested fields obtain defaults from their own property annotations and cannot declare default in #[property(...)] + --> tests/compile-fail/properties-view/view_nested_default.rs:29:9 + | +29 | / #[property(nested, default = unreachable!())] +30 | | inner: Inner<'_>, + | |________________________^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs b/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs new file mode 100644 index 0000000000..cbba6e50e1 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +fn parse_all( + _properties: &HashMap, + _key: &str, + _additional_keys: &[&str], + default: u64, +) -> iceberg::Result { + Ok(default) +} + +properties_view! { + struct ParsePropertiesWithoutAdditionalKeys { + #[property(key = "value", default = 1, parse_properties_with = parse_all)] + value: u64, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.stderr new file mode 100644 index 0000000000..89a2af6048 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.stderr @@ -0,0 +1,6 @@ +error: parse_properties_with requires additional_keys in #[property(...)] + --> tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs:33:9 + | +33 | / #[property(key = "value", default = 1, parse_properties_with = parse_all)] +34 | | value: u64, + | |__________________^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.rs b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.rs new file mode 100644 index 0000000000..8b32a39588 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +properties_view! { + struct PrefixDefault { + #[property(prefix = "values.", default = HashMap::new())] + values: HashMap, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.stderr new file mode 100644 index 0000000000..560ae271b2 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.stderr @@ -0,0 +1,14 @@ +error: prefix fields collect matching properties and cannot declare default in #[property(...)] + --> tests/compile-fail/properties-view/view_prefix_default.rs:24:9 + | +24 | / #[property(prefix = "values.", default = HashMap::new())] +25 | | values: HashMap, + | |____________________________________^ + +warning: unused import: `std::collections::HashMap` + --> tests/compile-fail/properties-view/view_prefix_default.rs:18:5 + | +18 | use std::collections::HashMap; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.rs b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.rs new file mode 100644 index 0000000000..05b6952f34 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct PrefixNonMap { + #[property(prefix = "values.")] + value: u64, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.stderr new file mode 100644 index 0000000000..534e925890 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.stderr @@ -0,0 +1,5 @@ +error: property prefix fields must have type HashMap + --> tests/compile-fail/properties-view/view_prefix_non_map.rs:23:16 + | +23 | value: u64, + | ^^^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.rs b/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.rs new file mode 100644 index 0000000000..33dfd284de --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.rs @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct TupleView(#[property(key = "value", default = 1)] u64); +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.stderr new file mode 100644 index 0000000000..415ff5f276 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.stderr @@ -0,0 +1,5 @@ +error: properties_view! requires a struct-shaped declaration with named fields + --> tests/compile-fail/properties-view/view_tuple_declaration.rs:21:12 + | +21 | struct TupleView(#[property(key = "value", default = 1)] u64); + | ^^^^^^^^^ diff --git a/crates/property-macro/tests/compile_fail.rs b/crates/property-macro/tests/compile_fail.rs index 3d692ce893..52749a1cbf 100644 --- a/crates/property-macro/tests/compile_fail.rs +++ b/crates/property-macro/tests/compile_fail.rs @@ -16,6 +16,11 @@ // under the License. #[test] -fn reports_invalid_property_annotations() { - trybuild::TestCases::new().compile_fail("tests/compile-fail/*.rs"); +fn reports_invalid_properties_derive_declarations() { + trybuild::TestCases::new().compile_fail("tests/compile-fail/derive-properties/*.rs"); +} + +#[test] +fn reports_invalid_property_view_declarations() { + trybuild::TestCases::new().compile_fail("tests/compile-fail/properties-view/*.rs"); } diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties_derive.rs similarity index 100% rename from crates/property-macro/tests/properties.rs rename to crates/property-macro/tests/properties_derive.rs diff --git a/crates/property-macro/tests/properties_view.rs b/crates/property-macro/tests/properties_view.rs new file mode 100644 index 0000000000..2a6cb05804 --- /dev/null +++ b/crates/property-macro/tests/properties_view.rs @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::mem::size_of; + +use iceberg::{Error, ErrorKind}; +use iceberg_property_macro::properties_view; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FORMAT: &str = "write.format.default"; +const FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; +const DEPTH: &str = "dimensions.depth"; + +fn parse_dimensions( + properties: &HashMap, + width_key: &str, + additional_keys: &[&str], + default: (u64, u64, u64), +) -> iceberg::Result<(u64, u64, u64)> { + if additional_keys.len() != 2 { + return Err(Error::new( + ErrorKind::DataInvalid, + "dimensions require height and depth keys", + )); + } + let parse = |property_key: &str, default| { + properties + .get(property_key) + .map(|value| { + value + .parse::() + .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) + }) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok(( + parse(width_key, default.0)?, + parse(additional_keys[0], default.1)?, + parse(additional_keys[1], default.2)?, + )) +} + +fn parse_non_empty(value: &str) -> iceberg::Result { + let value = value.trim(); + if value.is_empty() { + Err(Error::new( + ErrorKind::DataInvalid, + "value must not be empty", + )) + } else { + Ok(value.to_string()) + } +} + +properties_view! { + #[derive(Debug)] + struct TestPropertiesView { + /// Maximum number of times to retry a commit. + #[property(key = RETRIES, default = 4, getter)] + retries: u64, + + #[property(key = OWNER, default = None, getter)] + owner: Option, + + #[property(key = FORMAT, default = "parquet", getter)] + format: String, + + #[property(key = FANOUT_ENABLED, default = true, getter)] + fanout_enabled: bool, + + #[property(prefix = COLUMN_FPP_PREFIX, getter)] + column_fpp: HashMap, + + #[property( + key = "location", + default = "default", + parse_with = parse_non_empty, + getter + )] + location: String, + + #[property( + key = WIDTH, + additional_keys = [HEIGHT, DEPTH], + default = (640, 480, 320), + parse_properties_with = parse_dimensions, + getter + )] + dimensions: (u64, u64, u64), + } +} + +properties_view! { + #[derive(Debug)] + struct CommitPropertiesView { + #[property(key = RETRIES, default = 4, getter)] + retries: u64, + } +} + +properties_view! { + #[derive(Debug)] + struct NestedPropertiesView { + #[property(nested, getter)] + commit: CommitPropertiesView<'_>, + } +} + +#[test] +fn property_view_is_only_a_reference_to_the_source_map() { + assert_eq!( + size_of::>(), + size_of::<&HashMap>() + ); +} + +#[test] +fn property_view_parses_only_the_requested_field() { + let raw = HashMap::from([ + (RETRIES.to_string(), "many".to_string()), + (OWNER.to_string(), "iceberg".to_string()), + (FORMAT.to_string(), "orc".to_string()), + (FANOUT_ENABLED.to_string(), "FALSE".to_string()), + (WIDTH.to_string(), "1920".to_string()), + (HEIGHT.to_string(), "1080".to_string()), + (DEPTH.to_string(), "720".to_string()), + ]); + let properties = TestPropertiesView::new(&raw); + + let error = properties.retries().unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.message().contains(RETRIES)); + assert_eq!(properties.owner().unwrap().as_deref(), Some("iceberg")); + assert_eq!(properties.format().unwrap(), "orc"); + assert!(!properties.fanout_enabled().unwrap()); + assert_eq!(properties.dimensions().unwrap(), (1920, 1080, 720)); +} + +#[test] +fn property_view_uses_defaults_and_supports_custom_parsers() { + let raw = HashMap::from([("location".to_string(), " path ".to_string())]); + let properties = TestPropertiesView::new(&raw); + + assert_eq!(properties.retries().unwrap(), 4); + assert_eq!(properties.owner().unwrap(), None); + assert_eq!(properties.format().unwrap(), "parquet"); + assert!(properties.fanout_enabled().unwrap()); + assert!(properties.column_fpp().unwrap().is_empty()); + assert_eq!(properties.location().unwrap(), "path"); +} + +#[test] +fn nested_property_views_borrow_the_same_source_map() { + let raw = HashMap::from([(RETRIES.to_string(), "9".to_string())]); + let commit = { + let properties = NestedPropertiesView::new(&raw); + properties.commit().unwrap() + }; + + assert_eq!(commit.retries().unwrap(), 9); +} + +#[test] +fn property_view_reports_errors_when_the_corresponding_getter_is_called() { + let prefixed_key = format!("{COLUMN_FPP_PREFIX}id"); + let raw = HashMap::from([ + (prefixed_key.clone(), "low".to_string()), + (format!("{COLUMN_FPP_PREFIX}category"), "0.01".to_string()), + ("location".to_string(), " ".to_string()), + ]); + let properties = TestPropertiesView::new(&raw); + + let prefix_error = properties.column_fpp().unwrap_err(); + assert!(prefix_error.message().contains(&prefixed_key)); + + let location_error = properties.location().unwrap_err(); + assert_eq!(location_error.message(), "value must not be empty"); + assert!(format!("{location_error}").contains("property: location")); +} From 5266f7dd3241b6b94843e85f10771d2452c8b874 Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Fri, 28 Aug 2026 18:36:25 +0800 Subject: [PATCH 07/74] refactor: parse table properties lazily (#3030) --- crates/iceberg/public-api.txt | 211 ++++---- .../iceberg/src/catalog/metadata_location.rs | 13 +- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/encryption/manager.rs | 5 +- crates/iceberg/src/spec/mod.rs | 1 - crates/iceberg/src/spec/table_metadata.rs | 115 +++- .../src/spec/table_metadata_builder.rs | 26 + crates/iceberg/src/spec/table_properties.rs | 491 ++++++++---------- .../src/transaction/expire_snapshots.rs | 23 +- crates/iceberg/src/transaction/mod.rs | 12 +- .../writer/file_writer/location_generator.rs | 25 +- .../src/writer/file_writer/parquet_writer.rs | 63 ++- .../src/writer/file_writer/rolling_writer.rs | 5 +- .../datafusion/src/physical_plan/write.rs | 19 +- 14 files changed, 524 insertions(+), 487 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 19f6c26968..4aeddb665a 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2729,7 +2729,7 @@ pub fn iceberg::spec::TableMetadata::sort_order_by_id(&self, sort_order_id: i64) pub fn iceberg::spec::TableMetadata::sort_orders_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator pub fn iceberg::spec::TableMetadata::statistics_for_snapshot(&self, snapshot_id: i64) -> core::option::Option<&iceberg::spec::StatisticsFile> pub fn iceberg::spec::TableMetadata::statistics_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator -pub fn iceberg::spec::TableMetadata::table_properties(&self) -> iceberg::Result +pub fn iceberg::spec::TableMetadata::table_properties(&self) -> iceberg::spec::TableProperties<'_> pub fn iceberg::spec::TableMetadata::uuid(&self) -> uuid::Uuid pub async fn iceberg::spec::TableMetadata::write_to(&self, file_io: &iceberg::io::FileIO, metadata_location: &iceberg::MetadataLocation) -> iceberg::Result<()> impl core::clone::Clone for iceberg::spec::TableMetadata @@ -2796,111 +2796,108 @@ impl core::clone::Clone for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::TableMetadataBuilder impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct iceberg::spec::TableProperties -impl iceberg::spec::TableProperties -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SCHEMA: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str -pub const iceberg::spec::TableProperties::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str -pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_PARTITION_SPEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_DEFAULT_SORT_ORDER: &str -pub const iceberg::spec::TableProperties::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_ENCRYPTION_KEY_ID: &str -pub const iceberg::spec::TableProperties::PROPERTY_FORMAT_VERSION: &str -pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_GC_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 -pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS: &str -pub const iceberg::spec::TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str -pub const iceberg::spec::TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str -pub const iceberg::spec::TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::PROPERTY_SNAPSHOT_COUNT: &str -pub const iceberg::spec::TableProperties::PROPERTY_UUID: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_DATA_LOCATION: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_METADATA_PATH: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str -pub const iceberg::spec::TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize -pub const iceberg::spec::TableProperties::RESERVED_PROPERTIES: [&str; 9] -pub fn iceberg::spec::TableProperties::data_encryption_key_size(&self) -> iceberg::Result -impl iceberg::spec::TableProperties -pub fn iceberg::spec::TableProperties::cdc_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::cdc_max_chunk_size(&self) -> usize -pub fn iceberg::spec::TableProperties::cdc_min_chunk_size(&self) -> usize -pub fn iceberg::spec::TableProperties::cdc_norm_level(&self) -> i32 -pub fn iceberg::spec::TableProperties::commit_max_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::commit_min_retry_wait_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::commit_num_retries(&self) -> usize -pub fn iceberg::spec::TableProperties::commit_total_retry_timeout_ms(&self) -> u64 -pub fn iceberg::spec::TableProperties::encryption_data_key_length(&self) -> usize -pub fn iceberg::spec::TableProperties::encryption_key_id(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::from_properties(properties: &std::collections::hash::map::HashMap) -> iceberg::Result -pub fn iceberg::spec::TableProperties::gc_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::max_ref_age_ms(&self) -> i64 -pub fn iceberg::spec::TableProperties::max_snapshot_age_ms(&self) -> i64 -pub fn iceberg::spec::TableProperties::metadata_compression_codec(&self) -> &iceberg::compression::CompressionCodec -pub fn iceberg::spec::TableProperties::min_snapshots_to_keep(&self) -> usize -pub fn iceberg::spec::TableProperties::parquet_compression_codec(&self) -> &iceberg::compression::CompressionCodec -pub fn iceberg::spec::TableProperties::parquet_dict_size_bytes(&self) -> usize -pub fn iceberg::spec::TableProperties::parquet_page_row_limit(&self) -> usize -pub fn iceberg::spec::TableProperties::parquet_page_size_bytes(&self) -> usize -pub fn iceberg::spec::TableProperties::parquet_row_group_size_bytes(&self) -> usize -pub fn iceberg::spec::TableProperties::write_data_location(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::write_datafusion_fanout_enabled(&self) -> bool -pub fn iceberg::spec::TableProperties::write_folder_storage_location(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::write_format_default(&self) -> &alloc::string::String -pub fn iceberg::spec::TableProperties::write_metadata_path(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::write_object_storage_location(&self) -> &core::option::Option -pub fn iceberg::spec::TableProperties::write_object_storage_partitioned_paths(&self) -> bool -pub fn iceberg::spec::TableProperties::write_target_file_size_bytes(&self) -> usize -impl core::convert::TryFrom<&std::collections::hash::map::HashMap> for iceberg::spec::TableProperties -pub type iceberg::spec::TableProperties::Error = iceberg::Error -pub fn iceberg::spec::TableProperties::try_from(properties: &std::collections::hash::map::HashMap) -> iceberg::Result -impl core::fmt::Debug for iceberg::spec::TableProperties -pub fn iceberg::spec::TableProperties::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::TableProperties<'properties> +impl iceberg::spec::TableProperties<'_> +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_NUM_RETRIES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SCHEMA: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_ID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_FILE_FORMAT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_PARTITION_SPEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_SORT_ORDER: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_KEY_ID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_FORMAT_VERSION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_GC_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_GC_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_REF_AGE_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_COMPRESSION_CODEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_SNAPSHOT_COUNT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_UUID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_DATA_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_METADATA_PATH: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::RESERVED_PROPERTIES: [&'static str; 9] +pub fn iceberg::spec::TableProperties<'_>::data_encryption_key_size(&self) -> iceberg::Result +impl<'properties> iceberg::spec::TableProperties<'properties> +pub fn iceberg::spec::TableProperties<'properties>::cdc_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_max_chunk_size(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_min_chunk_size(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_norm_level(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_max_retry_wait_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_min_retry_wait_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_num_retries(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_total_retry_timeout_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::encryption_data_key_length(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::encryption_key_id(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::gc_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::max_ref_age_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::max_snapshot_age_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::metadata_compression_codec(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::min_snapshots_to_keep(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::new(properties: &'properties std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableProperties<'properties>::parquet_compression_codec(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_dict_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_page_row_limit(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_page_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_row_group_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_data_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_datafusion_fanout_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_folder_storage_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_format_default(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_metadata_path(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_partitioned_paths(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_target_file_size_bytes(&self) -> iceberg::Result +impl<'properties> core::fmt::Debug for iceberg::spec::TableProperties<'properties> +pub fn iceberg::spec::TableProperties<'properties>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg::spec::UnboundPartitionField pub iceberg::spec::UnboundPartitionField::field_id: core::option::Option pub iceberg::spec::UnboundPartitionField::name: alloc::string::String @@ -3407,7 +3404,7 @@ pub async fn iceberg::writer::file_writer::ParquetWriter::close(self) -> iceberg pub async fn iceberg::writer::file_writer::ParquetWriter::write(&mut self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> pub struct iceberg::writer::file_writer::ParquetWriterBuilder impl iceberg::writer::file_writer::ParquetWriterBuilder -pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_table_properties(table_props: &iceberg::spec::TableProperties, schema: iceberg::spec::SchemaRef) -> iceberg::Result +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_table_properties(table_props: &iceberg::spec::TableProperties<'_>, schema: iceberg::spec::SchemaRef) -> iceberg::Result pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new_with_match_mode(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef, match_mode: iceberg::arrow::FieldMatchMode) -> Self pub fn iceberg::writer::file_writer::ParquetWriterBuilder::with_encryption_manager(self, encryption_manager: alloc::sync::Arc) -> Self diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index d5daa02b37..8240a326f3 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Display; use std::str::FromStr; use uuid::Uuid; use crate::compression::CompressionCodec; -use crate::spec::{TableMetadata, parse_metadata_file_compression}; +use crate::spec::TableMetadata; use crate::{Error, ErrorKind, Result}; /// Default folder name for metadata files under the table location, used when the @@ -43,12 +42,6 @@ pub struct MetadataLocation { } impl MetadataLocation { - /// Determines the compression codec from table properties. - /// Parse errors result in CompressionCodec::None. - fn compression_from_properties(properties: &HashMap) -> CompressionCodec { - parse_metadata_file_compression(properties).unwrap_or(CompressionCodec::None) - } - /// Creates a completely new metadata location starting at version 0, deriving the /// metadata directory and compression settings from the table metadata. /// Only used for creating a new table. For updates, see `with_next_version` and @@ -58,7 +51,7 @@ impl MetadataLocation { location: metadata.metadata_location()?, version: 0, id: Uuid::new_v4(), - compression_codec: Self::compression_from_properties(metadata.properties()), + compression_codec: metadata.metadata_compression_codec()?, }) } @@ -80,7 +73,7 @@ impl MetadataLocation { location: new_metadata.metadata_location()?, version: self.version, id: self.id, - compression_codec: Self::compression_from_properties(new_metadata.properties()), + compression_codec: new_metadata.metadata_compression_codec()?, }) } diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index 1611db8302..6e6997b214 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -61,7 +61,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> { } // Delete data files only if gc.enabled is true, to avoid corrupting shared tables - if metadata.table_properties()?.gc_enabled() { + if metadata.table_properties().gc_enabled()? { delete_data_files(io, &manifests_to_delete).await?; } diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs index 63e6438bd5..f54ef464d5 100644 --- a/crates/iceberg/src/encryption/manager.rs +++ b/crates/iceberg/src/encryption/manager.rs @@ -113,8 +113,9 @@ impl EncryptionManager { return Ok(None); } - let table_properties = metadata.table_properties()?; - let Some(table_key_id) = table_properties.encryption_key_id().as_deref() else { + let table_properties = metadata.table_properties(); + let encryption_key_id = table_properties.encryption_key_id()?; + let Some(table_key_id) = encryption_key_id.as_deref() else { if kms_client.is_some() { tracing::warn!( "KeyManagementClient provided but table does not have encryption.key-id set" diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs index b23ca1eda0..707ebbb630 100644 --- a/crates/iceberg/src/spec/mod.rs +++ b/crates/iceberg/src/spec/mod.rs @@ -50,7 +50,6 @@ pub use sort::*; pub use statistic_file::*; pub use table_metadata::*; pub(crate) use table_metadata_builder::FIRST_FIELD_ID; -pub(crate) use table_properties::parse_metadata_file_compression; pub use table_properties::*; pub use transform::*; pub(crate) use values::decimal_utils; diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index bb455a4a5c..92bb6ff4d8 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -35,7 +35,7 @@ pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataB use super::{ DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - TableProperties, parse_metadata_file_compression, + TableProperties, }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; @@ -370,9 +370,8 @@ impl TableMetadata { /// to the `metadata` subdirectory under the table location. pub fn metadata_location(&self) -> Result { Ok(self - .table_properties()? - .write_metadata_path() - .clone() + .table_properties() + .write_metadata_path()? .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME))) } @@ -385,14 +384,13 @@ impl TableMetadata { /// /// Returns an error if the compression codec property has an invalid value. pub fn metadata_compression_codec(&self) -> Result { - parse_metadata_file_compression(&self.properties) + self.table_properties().metadata_compression_codec() } - /// Returns typed table properties parsed from the raw properties map with defaults. - pub fn table_properties(&self) -> Result { - TableProperties::try_from(&self.properties).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Invalid table properties").with_source(e) - }) + /// Returns a typed view that parses each table property when its getter is called. + #[inline] + pub fn table_properties(&self) -> TableProperties<'_> { + TableProperties::new(&self.properties) } /// Return location of statistics files. @@ -499,7 +497,7 @@ impl TableMetadata { let json_data = serde_json::to_vec(self)?; // Check if compression codec from properties matches the one in metadata_location - let codec = parse_metadata_file_compression(&self.properties)?; + let codec = self.table_properties().metadata_compression_codec()?; if codec != metadata_location.compression_codec() { return Err(Error::new( @@ -4041,14 +4039,14 @@ mod tests { .unwrap() .metadata; - let props = metadata.table_properties().unwrap(); + let props = metadata.table_properties(); assert_eq!( - props.commit_num_retries(), + props.commit_num_retries().unwrap(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - props.write_target_file_size_bytes(), + props.write_target_file_size_bytes().unwrap(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); } @@ -4088,10 +4086,67 @@ mod tests { .unwrap() .metadata; - let props = metadata.table_properties().unwrap(); + let props = metadata.table_properties(); + + assert_eq!(props.commit_num_retries().unwrap(), 10); + assert_eq!(props.write_target_file_size_bytes().unwrap(), 1024); + } + + #[test] + fn test_deserialize_metadata_defers_invalid_table_property_errors() { + let invalid_retries = "not_a_number"; + let invalid_codec = "unknown"; + let target_file_size = "1024"; + + for file_name in [ + "TableMetadataV1Valid.json", + "TableMetadataV2ValidMinimal.json", + "TableMetadataV3ValidMinimal.json", + ] { + let path = format!("testdata/table_metadata/{file_name}"); + let mut json: serde_json::Value = + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); + json["properties"] = serde_json::json!({ + (TableProperties::PROPERTY_COMMIT_NUM_RETRIES): invalid_retries, + (TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC): invalid_codec, + (TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES): target_file_size, + }); + + let metadata: TableMetadata = serde_json::from_value(json).unwrap(); + assert_eq!( + metadata + .properties() + .get(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + .map(String::as_str), + Some(invalid_retries) + ); + + let table_properties = metadata.table_properties(); + let error = table_properties.commit_num_retries().unwrap_err(); + assert!( + error + .message() + .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 1024 + ); + let error = table_properties.metadata_compression_codec().unwrap_err(); + assert!( + format!("{error}").contains(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + ); - assert_eq!(props.commit_num_retries(), 10); - assert_eq!(props.write_target_file_size_bytes(), 1024); + let serialized = serde_json::to_value(metadata).unwrap(); + assert_eq!( + serialized["properties"][TableProperties::PROPERTY_COMMIT_NUM_RETRIES], + invalid_retries + ); + assert_eq!( + serialized["properties"][TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC], + invalid_codec + ); + } } #[test] @@ -4103,10 +4158,16 @@ mod tests { .build() .unwrap(); - let properties = HashMap::from([( - "commit.retry.num-retries".to_string(), - "not_a_number".to_string(), - )]); + let properties = HashMap::from([ + ( + TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), + "not_a_number".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "1024".to_string(), + ), + ]); let metadata = TableMetadataBuilder::new( schema, @@ -4121,9 +4182,17 @@ mod tests { .unwrap() .metadata; - let err = metadata.table_properties().unwrap_err(); + let table_properties = metadata.table_properties(); + let err = table_properties.commit_num_retries().unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("Invalid table properties")); + assert!( + err.message() + .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 1024 + ); } #[test] diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 3191d6c13c..6467014ab8 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -2095,6 +2095,32 @@ mod tests { assert_eq!(build_result.metadata.metadata_log.len(), 0); } + #[test] + fn test_table_properties_view_reflects_metadata_updates() { + let property = TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(); + let metadata = builder_without_changes(FormatVersion::V2) + .set_properties(HashMap::from([(property.clone(), "7".to_string())])) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.table_properties().commit_num_retries().unwrap(), 7); + + let metadata = metadata + .into_builder(None) + .remove_properties(&[property]) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!( + metadata.table_properties().commit_num_retries().unwrap(), + TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT + ); + } + #[test] fn test_no_metadata_log_entry_for_no_previous_location() { // Used for first commit after stage-creation of tables diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 7ed1c032d5..b927d36386 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; -use iceberg_property_macro::Properties; +use iceberg_property_macro::properties_view; use crate::compression::CompressionCodec; use crate::encryption::AesKeySize; @@ -32,29 +32,6 @@ fn parse_location_property(path: &str) -> Result { Ok(strip_trailing_slash(path).to_string()) } -/// Parse compression codec for metadata files from table properties. -/// Retrieves the compression codec property, applies defaults, and parses the value. -/// Only "none" (or empty string) and "gzip" are supported for metadata compression. -/// -/// # Arguments -/// -/// * `properties` - HashMap containing table properties -/// -/// # Errors -/// -/// Returns an error if the codec is not "none", "", or "gzip" (case-insensitive). -/// Lz4 and Zstd are not supported for metadata file compression. -pub(crate) fn parse_metadata_file_compression( - properties: &HashMap, -) -> Result { - let value = properties - .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) - .map(|s| s.as_str()) - .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); - - parse_metadata_compression(value) -} - fn parse_metadata_compression(value: &str) -> Result { // Handle empty string as None if value.is_empty() { @@ -139,8 +116,9 @@ fn parse_parquet_compression( }) } -/// TableProperties that contains the properties of a table. -#[derive(Debug, Properties)] +properties_view! { +/// Typed view over the properties of a table. +#[derive(Debug)] pub struct TableProperties { /// The number of times to retry a commit. #[property( @@ -359,8 +337,9 @@ pub struct TableProperties { )] write_object_storage_partitioned_paths: bool, } +} -impl TableProperties { +impl TableProperties<'_> { /// Reserved table property for table format version. /// /// Iceberg will default a new table's format version to the latest stable and recommended @@ -370,32 +349,33 @@ impl TableProperties { /// If this table property exists when creating a table, the table will use the specified format /// version. If a table updates this property, it will try to upgrade to the specified format /// version. - pub const PROPERTY_FORMAT_VERSION: &str = "format-version"; + pub const PROPERTY_FORMAT_VERSION: &'static str = "format-version"; /// Reserved table property for table UUID. - pub const PROPERTY_UUID: &str = "uuid"; + pub const PROPERTY_UUID: &'static str = "uuid"; /// Reserved table property for the total number of snapshots. - pub const PROPERTY_SNAPSHOT_COUNT: &str = "snapshot-count"; + pub const PROPERTY_SNAPSHOT_COUNT: &'static str = "snapshot-count"; /// Reserved table property for current snapshot summary. - pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str = "current-snapshot-summary"; + pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str = "current-snapshot-summary"; /// Reserved table property for current snapshot id. - pub const PROPERTY_CURRENT_SNAPSHOT_ID: &str = "current-snapshot-id"; + pub const PROPERTY_CURRENT_SNAPSHOT_ID: &'static str = "current-snapshot-id"; /// Reserved table property for current snapshot timestamp. - pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str = "current-snapshot-timestamp-ms"; + pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str = "current-snapshot-timestamp-ms"; /// Reserved table property for the JSON representation of current schema. - pub const PROPERTY_CURRENT_SCHEMA: &str = "current-schema"; + pub const PROPERTY_CURRENT_SCHEMA: &'static str = "current-schema"; /// Reserved table property for the JSON representation of current(default) partition spec. - pub const PROPERTY_DEFAULT_PARTITION_SPEC: &str = "default-partition-spec"; + pub const PROPERTY_DEFAULT_PARTITION_SPEC: &'static str = "default-partition-spec"; /// Reserved table property for the JSON representation of current(default) sort order. - pub const PROPERTY_DEFAULT_SORT_ORDER: &str = "default-sort-order"; + pub const PROPERTY_DEFAULT_SORT_ORDER: &'static str = "default-sort-order"; /// Property key for max number of previous versions to keep. - pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str = + pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str = "write.metadata.previous-versions-max"; /// Default value for max number of previous versions to keep. pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100; /// Property key for max number of partitions to keep summary stats for. - pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit"; + pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str = + "write.summary.partition-limit"; /// Default value for the max number of partitions to keep summary stats for. pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0; @@ -403,7 +383,7 @@ impl TableProperties { /// /// Reserved table properties are only used to control behaviors when creating or updating a /// table. The value of these properties are not persisted as a part of the table metadata. - pub const RESERVED_PROPERTIES: [&str; 9] = [ + pub const RESERVED_PROPERTIES: [&'static str; 9] = [ Self::PROPERTY_FORMAT_VERSION, Self::PROPERTY_UUID, Self::PROPERTY_SNAPSHOT_COUNT, @@ -416,88 +396,91 @@ impl TableProperties { ]; /// Property key for number of commit retries. - pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; + pub const PROPERTY_COMMIT_NUM_RETRIES: &'static str = "commit.retry.num-retries"; /// Default value for number of commit retries. pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; /// Property key for minimum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; + pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str = "commit.retry.min-wait-ms"; /// Default value for minimum wait time (ms) between retries. pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; /// Property key for maximum wait time (ms) between retries. - pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; + pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str = "commit.retry.max-wait-ms"; /// Default value for maximum wait time (ms) between retries. pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute /// Property key for total maximum retry time (ms). - pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; + pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str = "commit.retry.total-timeout-ms"; /// Default value for total maximum retry time (ms). pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes /// Default file format for data files - pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = "write.format.default"; + pub const PROPERTY_DEFAULT_FILE_FORMAT: &'static str = "write.format.default"; /// Default file format for delete files - pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str = "write.delete.format.default"; + pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str = "write.delete.format.default"; /// Default value for data file format - pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; + pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str = "parquet"; /// Target file size for newly written files. - pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str = "write.target-file-size-bytes"; /// Default target file size pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB /// Base location for metadata files (manifests, manifest lists, table metadata). /// When unset, metadata files default to the `metadata` directory under the table /// location. - pub const PROPERTY_WRITE_METADATA_PATH: &str = "write.metadata.path"; + pub const PROPERTY_WRITE_METADATA_PATH: &'static str = "write.metadata.path"; /// Compression codec for metadata files (JSON) - pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; + pub const PROPERTY_METADATA_COMPRESSION_CODEC: &'static str = + "write.metadata.compression-codec"; /// Default metadata compression codec - uncompressed - pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; + pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str = "none"; /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data). /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient). - pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str = + "write.datafusion.fanout.enabled"; /// Default value for fanout writer enabled pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; /// Property key for enabling garbage collection on drop. /// When set to `false`, data files will not be deleted when a table is dropped. /// Defaults to `true`. - pub const PROPERTY_GC_ENABLED: &str = "gc.enabled"; + pub const PROPERTY_GC_ENABLED: &'static str = "gc.enabled"; /// Default value for gc.enabled pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true; /// Property key for the default maximum age of a snapshot to keep when expiring snapshots. - pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &str = "history.expire.max-snapshot-age-ms"; + pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str = "history.expire.max-snapshot-age-ms"; /// Default value for history.expire.max-snapshot-age-ms (5 days). pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; /// Property key for the default minimum number of snapshots to keep when expiring snapshots. - pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &str = "history.expire.min-snapshots-to-keep"; + pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str = "history.expire.min-snapshots-to-keep"; /// Default value for history.expire.min-snapshots-to-keep. pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; /// Property key for the default maximum age of a snapshot reference to keep when expiring. - pub const PROPERTY_MAX_REF_AGE_MS: &str = "history.expire.max-ref-age-ms"; + pub const PROPERTY_MAX_REF_AGE_MS: &'static str = "history.expire.max-ref-age-ms"; /// Default value for history.expire.max-ref-age-ms (effectively never expire refs). pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; /// Enable content-defined chunking with parquet defaults (or per-property overrides). - pub const PROPERTY_PARQUET_CDC_ENABLED: &str = "write.parquet.content-defined-chunking.enabled"; + pub const PROPERTY_PARQUET_CDC_ENABLED: &'static str = + "write.parquet.content-defined-chunking.enabled"; /// Default value for content-defined chunking enabled. pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false; /// Minimum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &str = + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str = "write.parquet.content-defined-chunking.min-chunk-size"; /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`. pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; /// Maximum chunk size in bytes for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &str = + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str = "write.parquet.content-defined-chunking.max-chunk-size"; /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`. pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; /// Normalization level (gearhash bit adjustment) for content-defined chunking. - pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &str = + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str = "write.parquet.content-defined-chunking.norm-level"; /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; @@ -506,49 +489,50 @@ impl TableProperties { /// `lz4`, `lz4_raw`, `brotli`, `lzo`, `uncompressed`). The codec name is /// parsed into a [`CompressionCodec`] when properties are parsed; the level's /// range is validated when the writer is built. - pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &str = "write.parquet.compression-codec"; + pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str = "write.parquet.compression-codec"; /// Default Parquet compression codec. - pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &str = "zstd"; + pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str = "zstd"; /// Compression level for Parquet data files, for codecs that take one /// (`gzip`, `zstd`, `brotli`). When unset, the codec's default level is used. - pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &str = "write.parquet.compression-level"; + pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str = "write.parquet.compression-level"; /// Approximate maximum size of a Parquet row group in bytes. - pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &str = "write.parquet.row-group-size-bytes"; + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str = + "write.parquet.row-group-size-bytes"; /// Default Parquet row group size in bytes. pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; /// Approximate maximum size of a Parquet data page in bytes. - pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &str = "write.parquet.page-size-bytes"; + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str = "write.parquet.page-size-bytes"; /// Default Parquet page size in bytes. pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; /// Maximum number of rows per Parquet data page. - pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &str = "write.parquet.page-row-limit"; + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str = "write.parquet.page-row-limit"; /// Default Parquet page row limit. pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000; /// Approximate maximum size of the Parquet dictionary page in bytes. - pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &str = "write.parquet.dict-size-bytes"; + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str = "write.parquet.dict-size-bytes"; /// Default Parquet dictionary page size in bytes. pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; /// Property key for the master key id used to encrypt the table's manifest /// list and data files as defined in . - pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; + pub const PROPERTY_ENCRYPTION_KEY_ID: &'static str = "encryption.key-id"; /// Property key for the encryption data encryption key (DEK) length in bytes. - pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str = "encryption.data-key-length"; /// Default value for the encryption DEK length (16 bytes = AES-128). pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; /// Property key for the base directory for data files - pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path"; + pub const PROPERTY_WRITE_DATA_LOCATION: &'static str = "write.data.path"; /// Property key for deprecated [`TableProperties::write_folder_storage_location`] - pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; + pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str = "write.folder-storage.path"; /// Property key for deprecated object storage path, kept as a fallback for compatibility. - pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path"; + pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str = "write.object-storage.path"; /// Property key for controlling whether partition values are included in object storage paths. - pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str = + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str = "write.object-storage.partitioned-paths"; /// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS`] pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; @@ -556,15 +540,7 @@ impl TableProperties { /// The AES key size to use when generating data encryption keys, derived /// from `encryption.data-key-length`. pub fn data_encryption_key_size(&self) -> Result { - AesKeySize::from_key_length(self.encryption_data_key_length) - } -} - -impl TryFrom<&HashMap> for TableProperties { - type Error = Error; - - fn try_from(properties: &HashMap) -> Result { - Self::from_properties(properties) + AesKeySize::from_key_length(self.encryption_data_key_length()?) } } @@ -576,46 +552,46 @@ mod tests { #[test] fn test_table_properties_default() { let props = HashMap::new(); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = TableProperties::new(&props); assert_eq!( - table_properties.commit_num_retries, + table_properties.commit_num_retries().unwrap(), TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT ); assert_eq!( - table_properties.commit_min_retry_wait_ms, + table_properties.commit_min_retry_wait_ms().unwrap(), TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT ); assert_eq!( - table_properties.commit_max_retry_wait_ms, + table_properties.commit_max_retry_wait_ms().unwrap(), TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT ); assert_eq!( - table_properties.write_format_default, + table_properties.write_format_default().unwrap(), TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string() ); assert_eq!( - table_properties.write_target_file_size_bytes, + table_properties.write_target_file_size_bytes().unwrap(), TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); // Test compression defaults (none means CompressionCodec::None) assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::None ); assert_eq!( - table_properties.gc_enabled, + table_properties.gc_enabled().unwrap(), TableProperties::PROPERTY_GC_ENABLED_DEFAULT ); assert_eq!( - table_properties.max_snapshot_age_ms, + table_properties.max_snapshot_age_ms().unwrap(), TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT ); assert_eq!( - table_properties.min_snapshots_to_keep, + table_properties.min_snapshots_to_keep().unwrap(), TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT ); assert_eq!( - table_properties.max_ref_age_ms, + table_properties.max_ref_age_ms().unwrap(), TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT ); } @@ -636,20 +612,27 @@ mod tests { "5678".to_string(), ), ]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.max_snapshot_age_ms, 1234); - assert_eq!(table_properties.min_snapshots_to_keep, 7); - assert_eq!(table_properties.max_ref_age_ms, 5678); + let table_properties = TableProperties::new(&props); + assert_eq!(table_properties.max_snapshot_age_ms().unwrap(), 1234); + assert_eq!(table_properties.min_snapshots_to_keep().unwrap(), 7); + assert_eq!(table_properties.max_ref_age_ms().unwrap(), 5678); } #[test] fn test_table_properties_location_paths() { // Test unset. - let table_properties = TableProperties::try_from(&HashMap::new()).unwrap(); - assert_eq!(table_properties.write_metadata_path, None); - assert_eq!(table_properties.write_data_location, None); - assert_eq!(table_properties.write_folder_storage_location, None); - assert_eq!(table_properties.write_object_storage_location, None); + let raw_properties = HashMap::new(); + let table_properties = TableProperties::new(&raw_properties); + assert_eq!(table_properties.write_metadata_path().unwrap(), None); + assert_eq!(table_properties.write_data_location().unwrap(), None); + assert_eq!( + table_properties.write_folder_storage_location().unwrap(), + None + ); + assert_eq!( + table_properties.write_object_storage_location().unwrap(), + None + ); for key in [ TableProperties::PROPERTY_WRITE_METADATA_PATH, @@ -658,34 +641,48 @@ mod tests { TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION, ] { // Test empty paths are invalid and retain the property key as error context. - let error = - TableProperties::try_from(&HashMap::from([(key.to_string(), String::new())])) - .unwrap_err(); + let raw_properties = HashMap::from([(key.to_string(), String::new())]); + let table_properties = TableProperties::new(&raw_properties); + let error = match key { + TableProperties::PROPERTY_WRITE_METADATA_PATH => { + table_properties.write_metadata_path().unwrap_err() + } + TableProperties::PROPERTY_WRITE_DATA_LOCATION => { + table_properties.write_data_location().unwrap_err() + } + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => table_properties + .write_folder_storage_location() + .unwrap_err(), + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => table_properties + .write_object_storage_location() + .unwrap_err(), + _ => unreachable!(), + }; assert_eq!(error.kind(), ErrorKind::DataInvalid); assert!(format!("{error}").contains(key)); // Test all supported location properties share trailing-slash normalization. - let table_properties = TableProperties::try_from(&HashMap::from([( + let raw_properties = HashMap::from([( key.to_string(), "s3://other-bucket/custom-path/".to_string(), - )])) - .unwrap(); + )]); + let table_properties = TableProperties::new(&raw_properties); let parsed = match key { TableProperties::PROPERTY_WRITE_METADATA_PATH => { - table_properties.write_metadata_path.as_deref() + table_properties.write_metadata_path().unwrap() } TableProperties::PROPERTY_WRITE_DATA_LOCATION => { - table_properties.write_data_location.as_deref() + table_properties.write_data_location().unwrap() } TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => { - table_properties.write_folder_storage_location.as_deref() + table_properties.write_folder_storage_location().unwrap() } TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => { - table_properties.write_object_storage_location.as_deref() + table_properties.write_object_storage_location().unwrap() } _ => unreachable!(), }; - assert_eq!(parsed, Some("s3://other-bucket/custom-path")); + assert_eq!(parsed.as_deref(), Some("s3://other-bucket/custom-path")); } } @@ -695,9 +692,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "gzip".to_string(), )]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = TableProperties::new(&props); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::gzip_default() ); } @@ -708,9 +705,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "none".to_string(), )]); - let table_properties = TableProperties::try_from(&props).unwrap(); + let table_properties = TableProperties::new(&props); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::None ); } @@ -722,9 +719,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GZIP".to_string(), )]); - let table_properties = TableProperties::try_from(&props_upper).unwrap(); + let table_properties = TableProperties::new(&props_upper); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::gzip_default() ); @@ -733,9 +730,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "GzIp".to_string(), )]); - let table_properties = TableProperties::try_from(&props_mixed).unwrap(); + let table_properties = TableProperties::new(&props_mixed); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::gzip_default() ); @@ -744,9 +741,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), "NONE".to_string(), )]); - let table_properties = TableProperties::try_from(&props_none_upper).unwrap(); + let table_properties = TableProperties::new(&props_none_upper); assert_eq!( - table_properties.metadata_compression_codec, + table_properties.metadata_compression_codec().unwrap(), CompressionCodec::None ); } @@ -775,12 +772,18 @@ mod tests { "false".to_string(), ), ]); - let table_properties = TableProperties::try_from(&props).unwrap(); - assert_eq!(table_properties.commit_num_retries, 10); - assert_eq!(table_properties.commit_max_retry_wait_ms, 20); - assert_eq!(table_properties.write_format_default, "avro".to_string()); - assert_eq!(table_properties.write_target_file_size_bytes, 512); - assert!(!table_properties.gc_enabled); + let table_properties = TableProperties::new(&props); + assert_eq!(table_properties.commit_num_retries().unwrap(), 10); + assert_eq!(table_properties.commit_max_retry_wait_ms().unwrap(), 20); + assert_eq!( + table_properties.write_format_default().unwrap(), + "avro".to_string() + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 512 + ); + assert!(!table_properties.gc_enabled().unwrap()); } #[test] @@ -790,9 +793,10 @@ mod tests { "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_retries).unwrap_err(); + let table_properties = TableProperties::new(&invalid_retries); + let error = table_properties.commit_num_retries().unwrap_err(); assert!( - table_properties.to_string().contains( + error.to_string().contains( "Invalid value for commit.retry.num-retries: invalid digit found in string" ) ); @@ -801,9 +805,10 @@ mod tests { TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_min_wait).unwrap_err(); + let table_properties = TableProperties::new(&invalid_min_wait); + let error = table_properties.commit_min_retry_wait_ms().unwrap_err(); assert!( - table_properties.to_string().contains( + error.to_string().contains( "Invalid value for commit.retry.min-wait-ms: invalid digit found in string" ) ); @@ -812,9 +817,10 @@ mod tests { TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_max_wait).unwrap_err(); + let table_properties = TableProperties::new(&invalid_max_wait); + let error = table_properties.commit_max_retry_wait_ms().unwrap_err(); assert!( - table_properties.to_string().contains( + error.to_string().contains( "Invalid value for commit.retry.max-wait-ms: invalid digit found in string" ) ); @@ -823,8 +829,9 @@ mod tests { TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), "abc".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_target_size).unwrap_err(); - assert!(table_properties.to_string().contains( + let table_properties = TableProperties::new(&invalid_target_size); + let error = table_properties.write_target_file_size_bytes().unwrap_err(); + assert!(error.to_string().contains( "Invalid value for write.target-file-size-bytes: invalid digit found in string" )); @@ -832,12 +839,9 @@ mod tests { TableProperties::PROPERTY_GC_ENABLED.to_string(), "notabool".to_string(), )]); - let table_properties = TableProperties::try_from(&invalid_gc_enabled).unwrap_err(); - assert!( - table_properties - .to_string() - .contains("Invalid value for gc.enabled") - ); + let table_properties = TableProperties::new(&invalid_gc_enabled); + let error = table_properties.gc_enabled().unwrap_err(); + assert!(error.to_string().contains("Invalid value for gc.enabled")); } #[test] @@ -849,7 +853,9 @@ mod tests { TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), codec.to_string(), )]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = TableProperties::new(&props) + .metadata_compression_codec() + .unwrap_err(); let err_msg = err.to_string(); assert!( err_msg.contains(&format!("Invalid metadata compression codec: {codec}")), @@ -862,103 +868,11 @@ mod tests { } } - #[test] - fn test_parse_metadata_file_compression_valid() { - // Test with "none" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "none".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test with empty string - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test with "gzip" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "gzip".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test case insensitivity - "NONE" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "NONE".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - - // Test case insensitivity - "GZIP" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GZIP".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test case insensitivity - "GzIp" - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - "GzIp".to_string(), - )]); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::gzip_default() - ); - - // Test default when property is missing - let props = HashMap::new(); - assert_eq!( - parse_metadata_file_compression(&props).unwrap(), - CompressionCodec::None - ); - } - - #[test] - fn test_parse_metadata_file_compression_invalid() { - let invalid_codecs = ["lz4", "zstd", "snappy"]; - - for codec in invalid_codecs { - let props = HashMap::from([( - TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), - codec.to_string(), - )]); - let err = parse_metadata_file_compression(&props).unwrap_err(); - let err_msg = err.to_string(); - assert!( - err_msg.contains("Invalid metadata compression codec"), - "Expected error message to contain 'Invalid metadata compression codec', got: {err_msg}" - ); - assert!( - err_msg.contains("Only 'none' and 'gzip' are supported"), - "Expected error message to contain supported codecs, got: {err_msg}" - ); - } - } - #[test] fn test_cdc_disabled_by_default() { let props = HashMap::new(); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); } #[test] @@ -967,11 +881,11 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), "true".to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 0); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024); + assert_eq!(tp.cdc_norm_level().unwrap(), 0); } #[test] @@ -980,8 +894,8 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), "262144".to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); } #[test] @@ -1004,11 +918,11 @@ mod tests { "1".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 200000); - assert_eq!(tp.cdc_max_chunk_size, 900000); - assert_eq!(tp.cdc_norm_level, 1); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 200000); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 900000); + assert_eq!(tp.cdc_norm_level().unwrap(), 1); } #[test] @@ -1023,11 +937,11 @@ mod tests { "2".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.cdc_enabled); - assert_eq!(tp.cdc_min_chunk_size, 256 * 1024); - assert_eq!(tp.cdc_max_chunk_size, 1024 * 1024); - assert_eq!(tp.cdc_norm_level, 2); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024); + assert_eq!(tp.cdc_norm_level().unwrap(), 2); } #[test] @@ -1042,8 +956,8 @@ mod tests { "-2".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.cdc_norm_level, -2); + let tp = TableProperties::new(&props); + assert_eq!(tp.cdc_norm_level().unwrap(), -2); } #[test] @@ -1058,7 +972,9 @@ mod tests { "not_a_number".to_string(), ), ]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = TableProperties::new(&props) + .cdc_min_chunk_size() + .unwrap_err(); assert!( err.to_string().contains( "Invalid value for write.parquet.content-defined-chunking.min-chunk-size" @@ -1078,7 +994,7 @@ mod tests { "not_a_number".to_string(), ), ]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = TableProperties::new(&props).cdc_norm_level().unwrap_err(); assert!( err.to_string() .contains("Invalid value for write.parquet.content-defined-chunking.norm-level") @@ -1088,32 +1004,33 @@ mod tests { #[test] fn test_cdc_no_properties() { let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.cdc_enabled); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); } #[test] fn test_parquet_sizing_defaults() { - let tp = TableProperties::try_from(&HashMap::new()).unwrap(); + let props = HashMap::new(); + let tp = TableProperties::new(&props); // Default codec is zstd at its default level. assert_eq!( - tp.parquet_compression_codec, + tp.parquet_compression_codec().unwrap(), CompressionCodec::zstd_default() ); assert_eq!( - tp.parquet_row_group_size_bytes, + tp.parquet_row_group_size_bytes().unwrap(), TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT ); assert_eq!( - tp.parquet_page_size_bytes, + tp.parquet_page_size_bytes().unwrap(), TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT ); assert_eq!( - tp.parquet_page_row_limit, + tp.parquet_page_row_limit().unwrap(), TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT ); assert_eq!( - tp.parquet_dict_size_bytes, + tp.parquet_dict_size_bytes().unwrap(), TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT ); } @@ -1146,13 +1063,16 @@ mod tests { "131072".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); + let tp = TableProperties::new(&props); // Codec name and level are folded into a single CompressionCodec. - assert_eq!(tp.parquet_compression_codec, CompressionCodec::Gzip(4)); - assert_eq!(tp.parquet_row_group_size_bytes, 1048576); - assert_eq!(tp.parquet_page_size_bytes, 65536); - assert_eq!(tp.parquet_page_row_limit, 5000); - assert_eq!(tp.parquet_dict_size_bytes, 131072); + assert_eq!( + tp.parquet_compression_codec().unwrap(), + CompressionCodec::Gzip(4) + ); + assert_eq!(tp.parquet_row_group_size_bytes().unwrap(), 1048576); + assert_eq!(tp.parquet_page_size_bytes().unwrap(), 65536); + assert_eq!(tp.parquet_page_row_limit().unwrap(), 5000); + assert_eq!(tp.parquet_dict_size_bytes().unwrap(), 131072); } #[test] @@ -1161,7 +1081,9 @@ mod tests { TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(), "not_a_number".to_string(), )]); - let err = TableProperties::try_from(&props).unwrap_err(); + let err = TableProperties::new(&props) + .parquet_row_group_size_bytes() + .unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!( err.to_string() @@ -1187,8 +1109,12 @@ mod tests { TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), name.to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.parquet_compression_codec, expected, "codec {name}"); + let tp = TableProperties::new(&props); + assert_eq!( + tp.parquet_compression_codec().unwrap(), + expected, + "codec {name}" + ); } } @@ -1206,8 +1132,11 @@ mod tests { "5".to_string(), ), ]); - let tp = TableProperties::try_from(&props).unwrap(); - assert_eq!(tp.parquet_compression_codec, CompressionCodec::Snappy); + let tp = TableProperties::new(&props); + assert_eq!( + tp.parquet_compression_codec().unwrap(), + CompressionCodec::Snappy + ); } #[test] @@ -1220,8 +1149,8 @@ mod tests { TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), f.to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(!tp.write_object_storage_partitioned_paths); + let tp = TableProperties::new(&props); + assert!(!tp.write_object_storage_partitioned_paths().unwrap()); } for t in true_variants { @@ -1229,8 +1158,8 @@ mod tests { TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), t.to_string(), )]); - let tp = TableProperties::try_from(&props).unwrap(); - assert!(tp.write_object_storage_partitioned_paths); + let tp = TableProperties::new(&props); + assert!(tp.write_object_storage_partitioned_paths().unwrap()); } } } diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs index 2542dde68d..0a390d9829 100644 --- a/crates/iceberg/src/transaction/expire_snapshots.rs +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -97,7 +97,7 @@ impl ExpireSnapshotsAction { } /// Resolves the snapshots and refs to remove, following Java `RemoveSnapshots.internalApply`. - fn plan(&self, table: &Table, properties: &TableProperties) -> Result { + fn plan(&self, table: &Table, properties: &TableProperties<'_>) -> Result { // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. if self.retain_last == Some(0) { return Err(Error::new( @@ -111,21 +111,24 @@ impl ExpireSnapshotsAction { // When a knob is not set explicitly, fall back to the table's `history.expire.*` properties, // matching Java `RemoveSnapshots`' constructor. With the default `max-snapshot-age-ms` (5 // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. - let default_cutoff = self - .older_than_ms - .unwrap_or_else(|| now.saturating_sub(properties.max_snapshot_age_ms())); - let default_min_to_keep = self - .retain_last - .unwrap_or(properties.min_snapshots_to_keep()); + let default_cutoff = match self.older_than_ms { + Some(older_than_ms) => older_than_ms, + None => now.saturating_sub(properties.max_snapshot_age_ms()?), + }; + let default_min_to_keep = match self.retain_last { + Some(retain_last) => retain_last, + None => properties.min_snapshots_to_keep()?, + }; // Ref aging: `main` is always kept; any other ref whose head is older than its // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's // `computeRetainedRefs`. + let default_max_ref_age_ms = properties.max_ref_age_ms()?; let mut removed_ref_names: Vec = vec![]; let mut retained_refs: Vec<&SnapshotReference> = vec![]; for (ref_name, snapshot_ref) in &metadata.refs { if ref_name == MAIN_BRANCH - || !Self::ref_aged_out(metadata, snapshot_ref, now, properties.max_ref_age_ms()) + || !Self::ref_aged_out(metadata, snapshot_ref, now, default_max_ref_age_ms) { retained_refs.push(snapshot_ref); } else { @@ -299,10 +302,10 @@ struct ExpirePlan { impl TransactionAction for ExpireSnapshotsAction { async fn commit(self: Arc, table: &Table) -> Result { let metadata = table.metadata(); - let properties = metadata.table_properties()?; + let properties = metadata.table_properties(); // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). - if !properties.gc_enabled() { + if !properties.gc_enabled()? { return Err(Error::new( ErrorKind::DataInvalid, "Cannot expire snapshots: gc.enabled is false", diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 2345eb7845..3622471277 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -178,7 +178,7 @@ impl Transaction { return Ok(self.table); } - let table_props = self.table.metadata().table_properties()?; + let table_props = self.table.metadata().table_properties(); let backoff = Self::build_backoff(table_props)?; let tx = self; @@ -195,14 +195,14 @@ impl Transaction { .1 } - fn build_backoff(props: TableProperties) -> Result { + fn build_backoff(props: TableProperties<'_>) -> Result { Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms())) - .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms())) + .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms()?)) + .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms()?)) .with_total_delay(Some(Duration::from_millis( - props.commit_total_retry_timeout_ms(), + props.commit_total_retry_timeout_ms()?, ))) - .with_max_times(props.commit_num_retries()) + .with_max_times(props.commit_num_retries()?) .with_factor(2.0) .build()) } diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index b8a612dc1a..26207d3e1d 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicU64; use crate::Result; -use crate::spec::{DataFileFormat, PartitionKey, TableMetadata, TableProperties}; +use crate::spec::{DataFileFormat, PartitionKey, TableMetadata}; use crate::util::location::strip_trailing_slash; /// `LocationGenerator` used to generate the location of data file. @@ -67,11 +67,12 @@ impl DefaultLocationGenerator { /// `{table_location}/data`. pub fn new(table_metadata: &TableMetadata) -> Result { let table_location = strip_trailing_slash(table_metadata.location()); - let prop = TableProperties::try_from(table_metadata.properties())?; + let prop = table_metadata.table_properties(); + let write_data_location = prop.write_data_location()?; + let write_folder_storage_location = prop.write_folder_storage_location()?; let data_location = strip_trailing_slash( - prop.write_data_location() - .clone() - .or_else(|| prop.write_folder_storage_location().clone()) + write_data_location + .or(write_folder_storage_location) .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) .as_ref(), ) @@ -135,12 +136,14 @@ impl ObjectStorageLocationGenerator { /// `{table_location}/data`. pub fn new(table_metadata: &TableMetadata) -> Result { let table_location = strip_trailing_slash(table_metadata.location()); - let prop = TableProperties::try_from(table_metadata.properties())?; + let prop = table_metadata.table_properties(); + let write_data_location = prop.write_data_location()?; + let write_object_storage_location = prop.write_object_storage_location()?; + let write_folder_storage_location = prop.write_folder_storage_location()?; let storage_location = strip_trailing_slash( - prop.write_data_location() - .clone() - .or_else(|| prop.write_object_storage_location().clone()) - .or_else(|| prop.write_folder_storage_location().clone()) + write_data_location + .or(write_object_storage_location) + .or(write_folder_storage_location) .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) .as_ref(), ) @@ -154,7 +157,7 @@ impl ObjectStorageLocationGenerator { Some(path_context(table_location)) }; - let include_partition_paths = prop.write_object_storage_partitioned_paths(); + let include_partition_paths = prop.write_object_storage_partitioned_paths()?; Ok(Self { storage_location, diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 2d9ef2077c..968a7b7236 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -87,20 +87,27 @@ impl ParquetWriterBuilder { /// Build a `ParquetWriterBuilder` from Iceberg table properties and a /// schema, translating `write.parquet.*` settings into `WriterProperties` /// instead of using parquet-rs defaults. - pub fn from_table_properties(table_props: &TableProperties, schema: SchemaRef) -> Result { - let cdc = table_props.cdc_enabled().then_some(CdcOptions { - min_chunk_size: table_props.cdc_min_chunk_size(), - max_chunk_size: table_props.cdc_max_chunk_size(), - norm_level: table_props.cdc_norm_level(), - }); - let compression = parquet_compression(*table_props.parquet_compression_codec())?; + pub fn from_table_properties( + table_props: &TableProperties<'_>, + schema: SchemaRef, + ) -> Result { + let cdc = if table_props.cdc_enabled()? { + Some(CdcOptions { + min_chunk_size: table_props.cdc_min_chunk_size()?, + max_chunk_size: table_props.cdc_max_chunk_size()?, + norm_level: table_props.cdc_norm_level()?, + }) + } else { + None + }; + let compression = parquet_compression(table_props.parquet_compression_codec()?)?; let props = WriterProperties::builder() .set_content_defined_chunking(cdc) .set_compression(compression) - .set_max_row_group_bytes(Some(table_props.parquet_row_group_size_bytes())) - .set_data_page_size_limit(table_props.parquet_page_size_bytes()) - .set_data_page_row_count_limit(table_props.parquet_page_row_limit()) - .set_dictionary_page_size_limit(table_props.parquet_dict_size_bytes()) + .set_max_row_group_bytes(Some(table_props.parquet_row_group_size_bytes()?)) + .set_data_page_size_limit(table_props.parquet_page_size_bytes()?) + .set_data_page_row_count_limit(table_props.parquet_page_row_limit()?) + .set_dictionary_page_size_limit(table_props.parquet_dict_size_bytes()?) .build(); Ok(Self::new_with_match_mode(props, schema, FieldMatchMode::Id)) } @@ -1046,10 +1053,11 @@ mod tests { let output_file = file_io.new_output( location_gen.generate_location(None, &file_name_gen.generate_file_name()), )?; - let table_properties = table_props(HashMap::from([( + let raw_properties = HashMap::from([( TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), "test-key".to_string(), - )])); + )]); + let table_properties = TableProperties::new(&raw_properties); let mut parquet_writer = ParquetWriterBuilder::from_table_properties(&table_properties, iceberg_schema.clone())? .with_encryption_manager(make_encryption_manager("test-key")) @@ -2546,20 +2554,18 @@ mod tests { ) } - fn table_props(entries: HashMap) -> TableProperties { - TableProperties::try_from(&entries).unwrap() - } - #[test] fn test_from_table_properties_no_cdc_by_default() { - let tp = table_props(HashMap::new()); + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); let builder = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()).unwrap(); assert!(builder.props.content_defined_chunking().is_none()); } #[tokio::test] async fn test_from_table_properties_without_encryption_writes_plaintext() { - let tp = table_props(HashMap::new()); + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); let tmp = TempDir::new().unwrap(); let output = FileIO::new_with_fs() .new_output(format!("{}/plain.parquet", tmp.path().to_str().unwrap())) @@ -2589,7 +2595,7 @@ mod tests { // written file) keeps this a direct propagation check: every future // `write.parquet.*` option just adds an assertion on its corresponding // `WriterProperties` getter here. - let tp = table_props(HashMap::from([ + let raw_properties = HashMap::from([ ( TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), "true".to_string(), @@ -2606,7 +2612,8 @@ mod tests { TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), "2".to_string(), ), - ])); + ]); + let tp = TableProperties::new(&raw_properties); let tmp = TempDir::new().unwrap(); let output = FileIO::new_with_fs() @@ -2632,7 +2639,8 @@ mod tests { fn test_from_table_properties_sizing_defaults() { // With no properties set, the writer must use Iceberg's defaults (which // differ from parquet-rs's own defaults), not parquet-rs's. - let tp = table_props(HashMap::new()); + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) .unwrap() .props; @@ -2662,7 +2670,7 @@ mod tests { #[test] fn test_from_table_properties_sizing_and_compression_overrides() { - let tp = table_props(HashMap::from([ + let raw_properties = HashMap::from([ ( TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(), "1048576".to_string(), @@ -2687,7 +2695,8 @@ mod tests { TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(), "9".to_string(), ), - ])); + ]); + let tp = TableProperties::new(&raw_properties); let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) .unwrap() .props; @@ -2708,7 +2717,11 @@ mod tests { TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), "bogus".to_string(), )]); - let err = TableProperties::try_from(&entries).unwrap_err(); + let err = ParquetWriterBuilder::from_table_properties( + &TableProperties::new(&entries), + cdc_test_schema(), + ) + .unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!(err.to_string().contains("bogus")); } diff --git a/crates/iceberg/src/writer/file_writer/rolling_writer.rs b/crates/iceberg/src/writer/file_writer/rolling_writer.rs index f569436cb0..9a45cd0fb2 100644 --- a/crates/iceberg/src/writer/file_writer/rolling_writer.rs +++ b/crates/iceberg/src/writer/file_writer/rolling_writer.rs @@ -456,10 +456,11 @@ mod tests { let schema = make_test_schema()?; - let table_properties = TableProperties::try_from(&HashMap::from([( + let raw_properties = HashMap::from([( TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), "test-key".to_string(), - )]))?; + )]); + let table_properties = TableProperties::new(&raw_properties); let parquet_writer_builder = ParquetWriterBuilder::from_table_properties(&table_properties, Arc::new(schema))? .with_encryption_manager(make_encryption_manager("test-key")); diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 8e680b3d92..3d2ea32226 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -202,15 +202,14 @@ impl ExecutionPlan for IcebergWriteExec { let format_version = self.table.metadata().format_version(); // Get typed table properties - let table_props = self - .table - .metadata() - .table_properties() - .map_err(to_datafusion_error)?; + let table_props = self.table.metadata().table_properties(); // Check data file format - let file_format = DataFileFormat::from_str(table_props.write_format_default()) + let write_format_default = table_props + .write_format_default() .map_err(to_datafusion_error)?; + let file_format = + DataFileFormat::from_str(&write_format_default).map_err(to_datafusion_error)?; if file_format != DataFileFormat::Parquet { return Err(to_datafusion_error(Error::new( ErrorKind::FeatureUnsupported, @@ -231,7 +230,9 @@ impl ExecutionPlan for IcebergWriteExec { parquet_file_writer_builder = parquet_file_writer_builder.with_encryption_manager(encryption_manager.clone()); } - let target_file_size = table_props.write_target_file_size_bytes(); + let target_file_size = table_props + .write_target_file_size_bytes() + .map_err(to_datafusion_error)?; let file_io = self.table.file_io().clone(); // todo location_gen and file_name_gen should be configurable @@ -250,7 +251,9 @@ impl ExecutionPlan for IcebergWriteExec { let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create TaskWriter - let fanout_enabled = table_props.write_datafusion_fanout_enabled(); + let fanout_enabled = table_props + .write_datafusion_fanout_enabled() + .map_err(to_datafusion_error)?; let schema = self.table.metadata().current_schema().clone(); let partition_spec = self.table.metadata().default_partition_spec().clone(); let task_writer = TaskWriter::try_new( From 30574bf2e7a8b4659a9260405766e231d2d4e32f Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 28 Aug 2026 19:16:40 -0700 Subject: [PATCH 08/74] feat: synthesize _row_id metadata column on read (#3058) * Refs https://github.com/apache/iceberg-rust/issues/2879 Read-side synthesis of the v3 _row_id metadata column, similar to the _last_updated_sequence_number materialization. We do a per-row coalesce of the physically-stored column against a per-row positional fallback of first_row_id plus the row position, gated on first_row_id (null first_row_id nulls the whole column). The positional fallback consumes the RowNumber virtual column, which is now added whenever _row_id is synthesized. Added tests * Fix CI failure * Fix typo flagged by lint * Synthesize _row_id in a reader-level stream step Reject a name-only physical _row_id column unconditionally (it never threads), fixing a false reject of user columns named _row_id under positional fallback. Add tests for global position across row groups, survival through positional deletes, and name collision under fallback. * Fix pr feedback * Fix merge collision --- crates/iceberg/src/arrow/reader/mod.rs | 4 +- crates/iceberg/src/arrow/reader/pipeline.rs | 910 +++++++++++++++++- crates/iceberg/src/arrow/reader/projection.rs | 20 + .../iceberg/src/arrow/reader/row_lineage.rs | 286 ++++++ 4 files changed, 1183 insertions(+), 37 deletions(-) create mode 100644 crates/iceberg/src/arrow/reader/row_lineage.rs diff --git a/crates/iceberg/src/arrow/reader/mod.rs b/crates/iceberg/src/arrow/reader/mod.rs index b5f1f2dccf..495283f47f 100644 --- a/crates/iceberg/src/arrow/reader/mod.rs +++ b/crates/iceberg/src/arrow/reader/mod.rs @@ -41,11 +41,13 @@ mod positional_deletes; mod predicate_visitor; mod projection; mod row_filter; +mod row_lineage; pub use file_reader::ArrowFileReader; pub(crate) use options::ParquetReadOptions; use predicate_visitor::{CollectFieldIdVisitor, PredicateConverter}; use projection::{ - add_fallback_field_ids_to_arrow_schema, apply_name_mapping_to_arrow_schema, build_field_id_map, + add_fallback_field_ids_to_arrow_schema, apply_name_mapping_to_arrow_schema, + find_leaf_by_field_id, }; /// Builder to create ArrowReader diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index b954055a28..9b66bea4cc 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -32,9 +32,10 @@ use parquet::arrow::{ }; use parquet::encryption::decrypt::FileDecryptionProperties; +use super::row_lineage::synthesize_row_id_column; use super::{ ArrowFileReader, ArrowReader, ParquetReadOptions, add_fallback_field_ids_to_arrow_schema, - apply_name_mapping_to_arrow_schema, build_field_id_map, + apply_name_mapping_to_arrow_schema, find_leaf_by_field_id, }; use crate::arrow::build_partition_constant; use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; @@ -45,9 +46,10 @@ use crate::encryption::StandardKeyMetadata; use crate::error::Result; use crate::io::{FileIO, FileMetadata, FileRead}; use crate::metadata_columns::{ - RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_COL_NAME_POS, RESERVED_FIELD_ID_FILE, + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_COL_NAME_POS, + RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_FIELD_ID_PARTITION, - RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_SPEC_ID, is_metadata_field, + RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, RESERVED_FIELD_ID_SPEC_ID, is_metadata_field, }; use crate::scan::{ArrowRecordBatchStream, FileScanTask, FileScanTaskStream}; use crate::spec::{Datum, PartitionSpec, Struct}; @@ -220,8 +222,15 @@ impl FileScanTaskReader { }; let project_pos = task.project_field_ids().contains(&RESERVED_FIELD_ID_POS); + let project_row_id = task.project_field_ids().contains(&RESERVED_FIELD_ID_ROW_ID); - let arrow_metadata = if project_pos { + // The RowNumber virtual column materializes `_pos`. It is also the per-row + // positional fallback for `_row_id` (`first_row_id + pos`), so add it whenever + // `_row_id` is synthesized. A null `first_row_id` nulls the whole `_row_id` + // column, so nothing is synthesized and the column is not needed. + let need_row_number = project_pos || (project_row_id && task.first_row_id.is_some()); + + let arrow_metadata = if need_row_number { let row_number_field = Arc::new( Field::new(RESERVED_COL_NAME_POS, DataType::Int64, false) .with_metadata(HashMap::from([( @@ -259,15 +268,16 @@ impl FileScanTaskReader { .project_field_ids() .contains(&RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER); - // Parquet leaf index of the physical column, if present by embedded field id. - // `build_field_id_map` is all-or-nothing: a file mixing id-bearing and id-less - // columns yields `None` and is rejected below rather than coalesced, even if the - // physical column itself carries its reserved id. + // Parquet leaf index of the physically-stored column, resolved by its embedded + // reserved field id. `find_leaf_by_field_id` tolerates id-less leaves (e.g. a + // Variant column's internal metadata/value leaves, which the spec requires to have + // no id), so an unprojected variant alongside a metadata column with correct ID does + // not hide it. let phys_last_updated_seq_leaf = if project_last_updated_seq { - build_field_id_map(record_batch_stream_builder.parquet_schema())?.and_then(|m| { - m.get(&RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER) - .copied() - }) + find_leaf_by_field_id( + record_batch_stream_builder.parquet_schema(), + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ) } else { None }; @@ -289,6 +299,33 @@ impl FileScanTaskReader { let coalesce_last_updated_seq_leaf = phys_last_updated_seq_leaf .filter(|_| task.first_row_id.is_some() && task.data_sequence_number.is_some()); + let phys_row_id_leaf = if project_row_id { + find_leaf_by_field_id( + record_batch_stream_builder.parquet_schema(), + RESERVED_FIELD_ID_ROW_ID, + ) + } else { + None + }; + + // A column named `_row_id` that carries no embedded field id, in a file that DOES + // use embedded ids -- an unthreadable physically-stored `_row_id`, rejected below. + // Gated on `!use_position_fallback`: under positional fallback every column lacks an + // embedded id and synthetic ids are assigned by position, so a user column that + // happens to be named `_row_id` is real data, not a reserved metadata column. + let row_id_present_by_name_only = project_row_id + && !use_position_fallback + && phys_row_id_leaf.is_none() + && record_batch_stream_builder + .schema() + .fields() + .iter() + .any(|f| f.name() == RESERVED_COL_NAME_ROW_ID); + + // Read the physical column only when first_row_id is set. A null first_row_id + // nulls the whole column below (matching Java `ValueReaders.rowIds`). + let coalesce_row_id_leaf = phys_row_id_leaf.filter(|_| task.first_row_id.is_some()); + // Filter out metadata fields for Parquet projection (they don't exist in files) let project_field_ids_without_metadata: Vec = task .project_field_ids @@ -314,25 +351,34 @@ impl FileScanTaskReader { // which `get_arrow_projection_mask` maps to "read all columns" (so `COUNT(*)` still // gets a row count). Downgrade that to "read no data columns" when a row-count // source exists independently of the data columns: the RowNumber virtual column - // (installed above under `project_pos`) or a physical metadata leaf unioned in - // below. Pure-constant / `COUNT(*)` projections have neither and must keep reading - // all columns to preserve the row count. Any future physical metadata leaf (e.g. a - // `_row_id` read path) is likewise a row source. + // (installed above under `need_row_number`, which covers `_pos` and `_row_id` + // synthesis) or a physical `_last_updated_sequence_number` leaf unioned in below + // (that column does not install RowNumber, so it is a separate source). Pure-constant + // / `COUNT(*)` projections have neither and must keep reading all columns to preserve + // the row count. + // + // This runs BEFORE the union so the physical leaves are added onto a `none` base, + // pruning the read to just those leaves (`union` with an `all` base stays `all`). // - // This runs BEFORE the union so the physical leaf is added onto a `none` base, - // pruning the read to just that leaf (`union` with an `all` base stays `all`). + // Known follow-up: a `_row_id`-only (or `_last_updated_sequence_number`-only) + // projection with a null `first_row_id` has no row-count source here, so it still + // reads all columns just to emit an all-null column. Pruning that to a minimal + // row-count source is left as a future optimization. if project_field_ids_without_metadata.is_empty() - && (project_pos || coalesce_last_updated_seq_leaf.is_some()) + && (need_row_number || coalesce_last_updated_seq_leaf.is_some()) { projection_mask = ProjectionMask::none(record_batch_stream_builder.parquet_schema().num_columns()); } - // Union in the physical `_last_updated_sequence_number` column when we will - // coalesce it. The metadata field id is not in the task schema, so it can't be - // requested through `get_arrow_projection_mask` (which resolves ids against the - // task schema); add its Parquet leaf directly. - if let Some(leaf) = coalesce_last_updated_seq_leaf { + // Union in the physical leaves of any metadata columns we will coalesce. Their + // reserved field ids are not in the task schema, so they can't be requested through + // `get_arrow_projection_mask` (which resolves ids against the task schema); add + // their Parquet leaves directly. + for leaf in [coalesce_last_updated_seq_leaf, coalesce_row_id_leaf] + .into_iter() + .flatten() + { let phys_mask = ProjectionMask::leaves(record_batch_stream_builder.parquet_schema(), vec![leaf]); projection_mask.union(&phys_mask); @@ -421,6 +467,27 @@ impl FileScanTaskReader { }; } + if project_row_id { + // A name-only physical `_row_id` can't be threaded (synthesis keys the leaf by + // its reserved field id). Reject only when `first_row_id` is set; otherwise the + // leaf is never read and `_row_id` is nulled out downstream (matching Java + // `ValueReaders.rowIds`), so a pre-v3 file with a user column named `_row_id` + // reads back as null rather than erroring. + if task.first_row_id.is_some() && row_id_present_by_name_only { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Reading a physically-stored _row_id column without an embedded field id \ + is not supported", + )); + } + + // `_row_id` is synthesized downstream over the record-batch stream (see + // `row_lineage::synthesize_row_id_column`); the transformer only passes the + // resulting column through, like `_pos`. + record_batch_transformer_builder = + record_batch_transformer_builder.with_virtual_field(RESERVED_FIELD_ID_ROW_ID); + } + if let (Some(partition_spec), Some(partition_data)) = (task.partition_spec.clone(), task.partition.clone()) { @@ -597,17 +664,18 @@ impl FileScanTaskReader { } // Build the batch stream and send all the RecordBatches that it generates - // to the requester. - let record_batch_stream = - record_batch_stream_builder - .build()? - .map(move |batch| match batch { - Ok(batch) => { - // Process the record batch (type promotion, column reordering, virtual fields, etc.) - record_batch_transformer.process_record_batch(batch) - } - Err(err) => Err(err.into()), - }); + // to the requester. When `_row_id` is projected, synthesize it over the raw parquet + // batches (using the reader-produced `_pos` position) before the transformer, which + // then passes it through as a virtual field. + let first_row_id = task.first_row_id; + let record_batch_stream = record_batch_stream_builder.build()?.map(move |batch| { + let mut batch = batch.map_err(|err| -> Error { err.into() })?; + if project_row_id { + batch = synthesize_row_id_column(batch, first_row_id)?; + } + // Process the record batch (type promotion, column reordering, virtual fields, etc.) + record_batch_transformer.process_record_batch(batch) + }); Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream) } @@ -711,7 +779,8 @@ mod tests { use crate::arrow::test_utils::write_encrypted_parquet; use crate::io::FileIO; use crate::metadata_columns::{ - RESERVED_COL_NAME_POS, RESERVED_FIELD_ID_FILE, RESERVED_FIELD_ID_POS, + RESERVED_COL_NAME_POS, RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE, + RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, }; use crate::scan::{FileScanTask, FileScanTaskStream}; use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Schema, SchemaRef, Type}; @@ -1453,6 +1522,775 @@ mod tests { ); } + /// A scan task projecting `id` + `_row_id`, with the given `first_row_id`. + fn row_id_task(file_path: String, first_row_id: Option) -> FileScanTask { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1, RESERVED_FIELD_ID_ROW_ID]) + .with_first_row_id(first_row_id) + .with_case_sensitive(false) + .build() + } + + /// Asserts the logical per-row values of the `_row_id` column across all batches, + /// independent of the physical (run-end) encoding. + fn assert_row_id_column(batches: &[RecordBatch], expected: &[Option]) { + use arrow_array::cast::AsArray; + use arrow_cast::cast; + use arrow_schema::DataType; + + let mut actual = Vec::new(); + for batch in batches { + let col = batch + .column_by_name(RESERVED_COL_NAME_ROW_ID) + .expect("_row_id column should be present"); + let logical = cast(col, &DataType::Int64).unwrap(); + let values = logical.as_primitive::(); + for i in 0..values.len() { + actual.push((!values.is_null(i)).then(|| values.value(i))); + } + } + assert_eq!(actual, expected); + } + + /// A parquet field carrying the embedded `_row_id` field id. + fn physical_row_id_field() -> Field { + Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true).with_metadata(HashMap::from([ + ( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_ROW_ID.to_string(), + ), + ])) + } + + #[tokio::test] + async fn test_row_id_synthesized_from_first_row_id_and_pos() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let file_path = write_plain_parquet(dir, "row_id_synth.parquet", vec![], vec![]); + + // No physical column: every row is first_row_id + pos. + let task = row_id_task(file_path, Some(100)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]); + } + + #[tokio::test] + async fn test_row_id_physical_column_coalesced() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // A file that physically carries `_row_id`, as written when carrying rows forward + // across a rewrite: some rows have a stored value, some are null. + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_phys.parquet", + vec![physical_row_id_field()], + vec![id_col], + ); + + let task = row_id_task(file_path, Some(100)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Per-row value where non-null; first_row_id + pos (101) where null. + assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]); + } + + #[tokio::test] + async fn test_row_id_only_synthesis_reads_no_data_columns() { + // The common v3 case: a new-row file with `first_row_id` set and NO physically + // stored `_row_id`, projecting only `_row_id`. `_row_id` synthesis installs the + // RowNumber virtual column (via `need_row_number`), so the row count comes from it + // -- the scan must read no data columns, not fall back to reading everything. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let mut meta_only = metadata_projection_task( + write_parquet_with_wide_column(dir, "row_id_only.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![RESERVED_FIELD_ID_ROW_ID], + ); + meta_only.first_row_id = Some(100); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]); + + // A scan that also projects the wide data column must read materially more. + let mut with_data = metadata_projection_task( + write_parquet_with_wide_column(dir, "row_id_only_ref.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![2, RESERVED_FIELD_ID_ROW_ID], + ); + with_data.first_row_id = Some(100); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_row_id-only synthesis should read fewer bytes than a scan of the wide column: \ + {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_row_id_resolves_alongside_id_less_leaf() { + // A file with an id-less leaf (mimicking a Variant column's internal metadata/value + // leaves, which the spec requires to have no field id) plus a physical `_row_id` + // that carries its embedded id. The reserved id must still resolve -- an + // all-or-nothing field map would bail on the id-less leaf and wrongly reject the file. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let idless_field = Field::new("variant_internal", DataType::Utf8, true); + let idless_col = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef; + let row_id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_with_idless_leaf.parquet", + vec![idless_field, physical_row_id_field()], + vec![idless_col, row_id_col], + ); + + let task = row_id_task(file_path, Some(100)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Physical value where non-null; first_row_id + pos (101) where null. + assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]); + } + + #[tokio::test] + async fn test_row_id_and_last_updated_seq_co_projected() { + use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER; + + // Both lineage columns projected together over a file carrying both physical + // leaves. Each must materialize independently -- neither leaf's mask clobbers the + // other, and the two synthesized columns keep their own values. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let row_id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let seq_col = Arc::new(Int64Array::from(vec![Some(50), None, Some(70)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_and_seq.parquet", + vec![physical_row_id_field(), physical_last_updated_seq_field()], + vec![row_id_col, seq_col], + ); + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![ + 1, + RESERVED_FIELD_ID_ROW_ID, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]) + .with_first_row_id(Some(100)) + .with_data_sequence_number(Some(9)) + .with_case_sensitive(false) + .build(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // _row_id: physical value where non-null, else first_row_id + pos (101). + assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]); + // _last_updated_sequence_number: physical value where non-null, else data seq (9). + assert_last_updated_seq_column(&batches, &[Some(50), Some(9), Some(70)]); + } + + #[tokio::test] + async fn test_row_id_null_when_no_first_row_id() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Physically carries `_row_id`, but the file has a null first_row_id. + let id_col = Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_no_first.parquet", + vec![physical_row_id_field()], + vec![id_col], + ); + + // Null first_row_id: the whole column is null; the physical values are not read. + let task = row_id_task(file_path, None); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_row_id_column(&batches, &[None, None, None]); + } + + #[tokio::test] + async fn test_row_id_with_pos_column() { + use crate::metadata_columns::RESERVED_COL_NAME_POS; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_and_pos.parquet", + vec![physical_row_id_field()], + vec![id_col], + ); + + // Co-project `_pos` and `_row_id`. `_row_id` synthesis consumes the position, and + // `_pos` is also emitted -- the RowNumber column must be added once and the two + // must not interfere. + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID]) + .with_first_row_id(Some(100)) + .with_case_sensitive(false) + .build(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // `_row_id` coalesces correctly... + assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]); + // ...and `_pos` is the row position, not double-counted. + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + } + + #[tokio::test] + async fn test_row_id_mixed_files_share_schema() { + use arrow_select::concat::concat_batches; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // Three files in one scan exercising all three column paths, which must all + // produce the SAME Arrow type (plain Int64) or concatenation fails: + // - synthesis: first_row_id set, no physical column -> first_row_id + pos + // - null gate: no first_row_id -> null column + // - coalesce: first_row_id set, physical column present -> per-row + fallback + let synth = row_id_task( + write_plain_parquet(dir, "row_id_synth2.parquet", vec![], vec![]), + Some(42), + ); + let nulled = row_id_task( + write_plain_parquet(dir, "row_id_null2.parquet", vec![], vec![]), + None, + ); + let coalesced = row_id_task( + write_plain_parquet( + dir, + "row_id_coalesced2.parquet", + vec![physical_row_id_field()], + vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef], + ), + Some(50), + ); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![ + Ok(synth), + Ok(nulled), + Ok(coalesced), + ])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_eq!(batches.len(), 3); + let schema = batches[0].schema(); + concat_batches(&schema, &batches) + .expect("synthesis, null and coalesce files must share one column type"); + } + + #[tokio::test] + async fn test_row_id_present_by_name_without_id_unsupported() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Column present by name but WITHOUT the embedded field id. The transformer keys + // the source column by field id, so this shape can't be threaded and is rejected. + let id_field = Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true); + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = + write_plain_parquet(dir, "row_id_by_name.parquet", vec![id_field], vec![id_col]); + + let task = row_id_task(file_path, Some(100)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert!( + format!("{err}").contains("without an embedded field id"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_row_id_present_by_name_without_id_nulls_when_no_lineage() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Same name-only shape as the reject test above, but the file carries no row lineage + // (`first_row_id = None`), as a migrated pre-v3 file with a user column named + // `_row_id` would. The physical leaf is never read, so `_row_id` is nulled out + // rather than rejected (matching Java `ValueReaders.rowIds`). + let id_field = Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true); + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_by_name_no_lineage.parquet", + vec![id_field], + vec![id_col], + ); + + let task = row_id_task(file_path, None); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_row_id_column(&batches, &[None, None, None]); + } + + /// Builds a `row_id_task` (see above) that additionally carries a bound predicate, + /// so a `RowSelection` is applied when the reader has row selection enabled. + fn row_id_task_with_predicate( + file_path: String, + first_row_id: Option, + extra_project_field_ids: Vec, + predicate: crate::expr::Predicate, + ) -> FileScanTask { + use crate::expr::Bind; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let bound = predicate.bind(Arc::clone(&schema), false).unwrap(); + + let mut project_field_ids = vec![1]; + project_field_ids.extend(extra_project_field_ids); + project_field_ids.push(RESERVED_FIELD_ID_ROW_ID); + + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(project_field_ids) + .with_predicate(Some(bound)) + .with_first_row_id(first_row_id) + .with_case_sensitive(false) + .build() + } + + #[tokio::test] + async fn test_row_id_stable_under_row_selection() { + use crate::expr::Reference; + use crate::spec::Datum; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // id = [1, 2, 3]; drop the middle physical row via a predicate + row selection. + let file_path = write_plain_parquet(dir, "row_id_selection.parquet", vec![], vec![]); + + let task = row_id_task_with_predicate( + file_path, + Some(100), + vec![], + Reference::new("id").not_equal_to(Datum::int(2)), + ); + + // Row selection must be enabled for the predicate to produce a RowSelection. + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_row_selection_enabled(true) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // The survivors are physical rows 0 and 2, so their _row_id is first_row_id + the + // PHYSICAL position: [100, 102]. A dense output index would wrongly give [100, 101]. + assert_row_id_column(&batches, &[Some(100), Some(102)]); + } + + #[tokio::test] + async fn test_row_id_coalesce_stable_under_row_selection() { + use crate::expr::Reference; + use crate::spec::Datum; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Physical _row_id = [Some(5), None, Some(8)] over id = [1, 2, 3]. Dropping the + // middle row must keep the physical column and the RowNumber fallback row-aligned. + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_coalesce_selection.parquet", + vec![physical_row_id_field()], + vec![id_col], + ); + + let task = row_id_task_with_predicate( + file_path, + Some(100), + vec![], + Reference::new("id").not_equal_to(Datum::int(2)), + ); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_row_selection_enabled(true) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Rows 0 and 2 survive: their stored values (5, 8) pass through. The dropped + // row's null (which would have fallen back to 100 + 1) is gone -- proving the + // physical column and the positional fallback are filtered by the same selection. + assert_row_id_column(&batches, &[Some(5), Some(8)]); + } + + #[tokio::test] + async fn test_row_id_global_across_row_groups() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // 5 rows written with max_row_group_size = 2 -> 3 row groups. `_pos` must be the + // GLOBAL file position, so `_row_id` continues across row-group boundaries rather + // than restarting per group (which would silently duplicate ids). + let file_path = format!("{dir}/row_id_multi_rg.parquet"); + let field = Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + vec![1, 2, 3, 4, 5], + ))]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(2)) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let task = row_id_task(file_path, Some(0)); + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_row_id_column(&batches, &[Some(0), Some(1), Some(2), Some(3), Some(4)]); + } + + #[tokio::test] + async fn test_row_id_global_when_first_row_group_pruned() { + use parquet::file::reader::{FileReader, SerializedFileReader}; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // 6 rows with max_row_group_size = 2 -> 3 row groups. A byte-range split that prunes + // row group 0 reaches the reader via `with_row_groups()` -- a different path than the + // `RowSelection` cases above. The survivors must keep their GLOBAL positions (starting + // at 2), so a per-group RowNumber restart would surface as duplicate ids here. + let file_path = format!("{dir}/row_id_prune_rg0.parquet"); + let field = Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + vec![1, 2, 3, 4, 5, 6], + ))]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(2)) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // A byte range starting just past row group 0 prunes it (its midpoint falls below + // `start`) while keeping groups 1 and 2 (physical rows 2..6). + let metadata = SerializedFileReader::new(File::open(&file_path).unwrap()) + .unwrap() + .metadata() + .clone(); + assert_eq!(metadata.num_row_groups(), 3); + let start = 4 + metadata.row_group(0).compressed_size() as u64; + let file_size = std::fs::metadata(&file_path).unwrap().len(); + + let mut task = row_id_task(file_path, Some(100)); + task.start = start; + task.length = file_size - start; + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Groups 1 and 2 survive at physical positions 2..6 -> _row_id 102..106, not a + // per-group restart at 100. + assert_row_id_column(&batches, &[Some(102), Some(103), Some(104), Some(105)]); + } + + /// Writes a positional delete file (`file_path` + `pos` reserved columns) marking the + /// given `positions` of `data_file_path` as deleted. + fn write_positional_delete( + dir: &str, + name: &str, + data_file_path: &str, + positions: &[i64], + ) -> String { + use arrow_array::StringArray; + + let file_path_field = + Field::new("file_path", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2147483546".to_string(), + )])); + let pos_field = Field::new("pos", DataType::Int64, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2147483545".to_string(), + )])); + let arrow_schema = Arc::new(ArrowSchema::new(vec![file_path_field, pos_field])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(StringArray::from(vec![data_file_path; positions.len()])), + Arc::new(Int64Array::from(positions.to_vec())), + ]) + .unwrap(); + + let path = format!("{dir}/{name}"); + let file = File::create(&path).unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + path + } + + #[tokio::test] + async fn test_row_id_survives_positional_delete() { + use crate::scan::FileScanTaskDeleteFile; + use crate::spec::DataContentType; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // id = [1, 2, 3]; a positional delete drops the middle physical row (pos = 1). + let data_path = write_plain_parquet(dir, "row_id_posdel_data.parquet", vec![], vec![]); + let del_path = write_positional_delete(dir, "row_id_posdel.parquet", &data_path, &[1]); + + let mut task = row_id_task(data_path, Some(100)); + task.deletes = vec![FileScanTaskDeleteFile { + file_path: del_path.clone(), + file_size_in_bytes: std::fs::metadata(&del_path).unwrap().len(), + file_type: DataContentType::PositionDeletes, + partition_spec_id: 0, + equality_ids: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + record_count: None, + key_metadata: None, + }]; + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Survivors keep their ABSOLUTE positions -- [100, 102], not renumbered [100, 101]. + // Positional-delete selection reaches the reader via a different path than predicate + // selection, so this covers it explicitly. + assert_row_id_column(&batches, &[Some(100), Some(102)]); + } + + #[tokio::test] + async fn test_row_id_name_collision_under_positional_fallback() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // A file with NO embedded field ids (positional fallback) whose columns include one + // literally named `_row_id` (user data). Projecting `_row_id` must NOT be rejected as + // an unthreadable physical metadata column -- under fallback the reserved column is + // synthesized and the same-named user column is just data. + let file_path = format!("{dir}/fallback_row_id_name.parquet"); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true), + ])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Int64Array::from(vec![7i64, 8, 9])), + ]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let task = row_id_task(file_path, Some(100)); + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Not rejected; `_row_id` is synthesized as first_row_id + pos. + assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]); + } + #[tokio::test] async fn test_read_encrypted_parquet_with_wrong_key_fails() { let encryption_key = b"0123456789abcdef"; diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index 9fdc9fed65..f832a26521 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -312,6 +312,26 @@ pub(super) fn build_field_id_map( Ok(Some(column_map)) } +/// Finds the Parquet leaf column index carrying `field_id` by its embedded id. +/// +/// Unlike [`build_field_id_map`], a leaf without an embedded id does not abort the +/// search -- it is simply skipped. This tolerates files that legitimately mix id-bearing +/// and id-less leaves, e.g. a Variant column whose internal metadata/value leaves are +/// required by the spec to have no field id, alongside a reserved metadata column that +/// does carry its id. +pub(super) fn find_leaf_by_field_id( + parquet_schema: &SchemaDescriptor, + field_id: i32, +) -> Option { + parquet_schema.columns().iter().position(|col| { + matches!( + col.self_type(), + ParquetType::PrimitiveType { basic_info, .. } + if basic_info.has_id() && basic_info.id() == field_id + ) + }) +} + /// Build a fallback field ID map for Parquet files without embedded field IDs. /// /// Returns the number of primitive (leaf) columns in a Parquet type, recursing into groups. diff --git a/crates/iceberg/src/arrow/reader/row_lineage.rs b/crates/iceberg/src/arrow/reader/row_lineage.rs new file mode 100644 index 0000000000..ee8dda4e6a --- /dev/null +++ b/crates/iceberg/src/arrow/reader/row_lineage.rs @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reader-level synthesis of the v3 `_row_id` metadata column. +//! +//! `_row_id` is `first_row_id + pos`, overridden by a physically-stored `_row_id` where the +//! file carries one. The position comes from the reader-produced `_pos` (`RowNumber`) +//! column, which is the true global file position under filter pushdown, row-group pruning, +//! and page-index pruning -- so synthesis is done here, over the record-batch stream, using +//! that position rather than recomputing it. + +use std::sync::Arc; + +use arrow_arith::boolean::is_not_null; +use arrow_arith::numeric::add; +use arrow_array::{Array, ArrayRef, Int64Array, RecordBatch}; +use arrow_schema::{DataType, Field, Schema}; +use arrow_select::zip::zip; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + +use crate::metadata_columns::{ + RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, +}; +use crate::{Error, ErrorKind, Result}; + +/// Appends the synthesized `_row_id` column to `batch`. +/// +/// - `first_row_id == None`: the file carries no row lineage, so `_row_id` is all-null +/// (matching Java `ValueReaders.rowIds`). +/// - `first_row_id == Some(base)`: `_row_id` is the physically-stored value where the file +/// carries one and it is non-null, else `base + pos` (from the `_pos`/`RowNumber` column). +/// +/// A physically-stored `_row_id` leaf already in `batch` is replaced by the synthesized +/// column (its values are folded in via the coalesce), so the result carries exactly one +/// column tagged with the reserved `_row_id` field id. +pub(crate) fn synthesize_row_id_column( + batch: RecordBatch, + first_row_id: Option, +) -> Result { + let row_id: ArrayRef = match first_row_id { + None => Arc::new(Int64Array::new_null(batch.num_rows())), + Some(base) => { + let pos = column_by_field_id(&batch, RESERVED_FIELD_ID_POS).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "_row_id synthesis requires the _pos position column in the record batch", + ) + })?; + if pos.data_type() != &DataType::Int64 { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "_pos position column must be Int64, got {}", + pos.data_type() + ), + )); + } + + // base + pos, the fallback for every row. + let fallback = add(pos, &Int64Array::new_scalar(base))?; + match column_by_field_id(&batch, RESERVED_FIELD_ID_ROW_ID) { + Some(id) => { + if id.data_type() != &DataType::Int64 { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("_row_id source must be Int64, got {}", id.data_type()), + )); + } + + zip(&is_not_null(id)?, id, &fallback)? + } + None => fallback, + } + } + }; + + append_row_id(batch, row_id) +} + +/// Returns the batch column tagged with `field_id` (via `PARQUET_FIELD_ID_META_KEY`), if any. +fn column_by_field_id(batch: &RecordBatch, field_id: i32) -> Option<&ArrayRef> { + batch + .schema() + .fields() + .iter() + .position(|f| field_id_of(f) == Some(field_id)) + .map(|idx| batch.column(idx)) +} + +fn field_id_of(field: &Field) -> Option { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .and_then(|id| id.parse::().ok()) +} + +/// Rebuilds `batch` with `row_id` as its `_row_id` column: any existing `_row_id`-tagged +/// column is dropped (already folded into `row_id`) and the synthesized column is appended. +fn append_row_id(batch: RecordBatch, row_id: ArrayRef) -> Result { + let schema = batch.schema(); + let mut fields: Vec> = Vec::with_capacity(schema.fields().len() + 1); + let mut columns: Vec = Vec::with_capacity(schema.fields().len() + 1); + + for (idx, field) in schema.fields().iter().enumerate() { + if field_id_of(field) != Some(RESERVED_FIELD_ID_ROW_ID) { + fields.push(field.clone()); + columns.push(batch.column(idx).clone()); + } + } + + fields.push(Arc::new( + Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true).with_metadata( + [( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_ROW_ID.to_string(), + )] + .into(), + ), + )); + columns.push(row_id); + + Ok(RecordBatch::try_new( + Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())), + columns, + )?) +} + +#[cfg(test)] +mod tests { + use arrow_array::cast::AsArray; + use arrow_array::types::Int64Type; + use arrow_array::{Int32Array, Int64Array}; + + use super::*; + use crate::metadata_columns::RESERVED_COL_NAME_POS; + + fn field_with_id(name: &str, dt: DataType, id: i32) -> Arc { + Arc::new( + Field::new(name, dt, true) + .with_metadata([(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string())].into()), + ) + } + + /// A batch with `id`, a `_pos` column, and (when `physical` is `Some`) a physically + /// stored `_row_id` column — mirroring what the Parquet reader produces. + fn batch(pos: Vec, physical: Option>>) -> RecordBatch { + let n = pos.len() as i32; + let mut fields = vec![ + field_with_id("id", DataType::Int32, 1), + field_with_id( + RESERVED_COL_NAME_POS, + DataType::Int64, + RESERVED_FIELD_ID_POS, + ), + ]; + let mut columns: Vec = vec![ + Arc::new(Int32Array::from((0..n).collect::>())), + Arc::new(Int64Array::from(pos)), + ]; + if let Some(vals) = physical { + fields.push(field_with_id( + RESERVED_COL_NAME_ROW_ID, + DataType::Int64, + RESERVED_FIELD_ID_ROW_ID, + )); + columns.push(Arc::new(Int64Array::from(vals))); + } + RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap() + } + + fn row_ids(rb: &RecordBatch) -> Vec> { + let v = rb + .column_by_name(RESERVED_COL_NAME_ROW_ID) + .unwrap() + .as_primitive::(); + (0..v.len()) + .map(|i| (!v.is_null(i)).then(|| v.value(i))) + .collect() + } + + #[test] + fn pure_synthesis() { + let out = synthesize_row_id_column(batch(vec![0, 1, 2], None), Some(100)).unwrap(); + assert_eq!(row_ids(&out), vec![Some(100), Some(101), Some(102)]); + } + + #[test] + fn non_zero_start_position() { + let out = synthesize_row_id_column(batch(vec![10, 11, 12], None), Some(100)).unwrap(); + assert_eq!(row_ids(&out), vec![Some(110), Some(111), Some(112)]); + } + + #[test] + fn coalesce_with_physical_column() { + let out = synthesize_row_id_column( + batch(vec![0, 1, 2], Some(vec![Some(5), None, Some(8)])), + Some(100), + ) + .unwrap(); + // Physical value where non-null, else first_row_id + pos. + assert_eq!(row_ids(&out), vec![Some(5), Some(101), Some(8)]); + // The physical leaf was folded in -- exactly one _row_id column (id, _pos, _row_id). + assert_eq!(out.schema().fields().len(), 3); + } + + #[test] + fn all_physical_present() { + let out = synthesize_row_id_column( + batch(vec![0, 1, 2], Some(vec![Some(5), Some(6), Some(7)])), + Some(100), + ) + .unwrap(); + assert_eq!(row_ids(&out), vec![Some(5), Some(6), Some(7)]); + } + + #[test] + fn all_fallback() { + let out = + synthesize_row_id_column(batch(vec![0, 1, 2], Some(vec![None, None, None])), Some(50)) + .unwrap(); + assert_eq!(row_ids(&out), vec![Some(50), Some(51), Some(52)]); + } + + #[test] + fn null_first_row_id_is_all_null() { + let out = synthesize_row_id_column(batch(vec![0, 1, 2], None), None).unwrap(); + assert_eq!(row_ids(&out), vec![None, None, None]); + } + + #[test] + fn empty_batch() { + let out = synthesize_row_id_column(batch(vec![], None), Some(100)).unwrap(); + assert_eq!(out.num_rows(), 0); + assert_eq!(row_ids(&out), Vec::>::new()); + } + + #[test] + fn non_int64_physical_column_is_rejected() { + let fields = vec![ + field_with_id( + RESERVED_COL_NAME_POS, + DataType::Int64, + RESERVED_FIELD_ID_POS, + ), + field_with_id( + RESERVED_COL_NAME_ROW_ID, + DataType::Int32, + RESERVED_FIELD_ID_ROW_ID, + ), + ]; + let columns: Vec = vec![ + Arc::new(Int64Array::from(vec![0i64, 1, 2])), + Arc::new(Int32Array::from(vec![Some(5), None, Some(8)])), + ]; + let rb = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(); + + let err = synthesize_row_id_column(rb, Some(100)).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(format!("{err}").contains("_row_id source must be Int64")); + } + + #[test] + fn missing_pos_is_rejected() { + let fields = vec![field_with_id("id", DataType::Int32, 1)]; + let columns: Vec = vec![Arc::new(Int32Array::from(vec![0, 1, 2]))]; + let rb = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(); + + let err = synthesize_row_id_column(rb, Some(100)).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::Unexpected); + assert!(format!("{err}").contains("_pos position column")); + } +} From 6d7afdd2d1a0cc8e28ec1e7de853e8f08eef0443 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:52:22 +0800 Subject: [PATCH 09/74] chore(deps): Bump reqwest from 0.12.28 to 0.13.4 (#3116) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.28 to 0.13.4. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.12.28...v0.13.4) --- updated-dependencies: - dependency-name: reqwest dependency-version: 0.13.4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3802cc03ff..e06ed57b47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1489,7 +1489,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -3725,7 +3725,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -5963,7 +5963,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.4", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -6002,7 +6002,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.5.10", "tracing", "windows-sys 0.61.2", ] @@ -8661,7 +8661,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] From cb6e2a4607b284fd46e6b631073bb1c8be17ec7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:01:19 +0800 Subject: [PATCH 10/74] chore(deps): Bump taiki-e/install-action from 2.86.1 to 2.86.6 (#3115) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.86.1 to 2.86.6. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/288e746965032cfcc232e09af2daf5f23c14d780...6cd13508893c0e7eab5f273c2575d3859bd7229a) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.86.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 394781eb0b..1ef51475aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - name: Install lint tools # Keep versions in sync with the local install targets in Makefile. - uses: taiki-e/install-action@288e746965032cfcc232e09af2daf5f23c14d780 # v2.86.1 + uses: taiki-e/install-action@6cd13508893c0e7eab5f273c2575d3859bd7229a # v2.86.6 with: tool: taplo@0.9.3,cargo-machete@0.7.0 @@ -209,7 +209,7 @@ jobs: - name: Install cargo-nextest if: matrix.test-suite.name == 'default' - uses: taiki-e/install-action@288e746965032cfcc232e09af2daf5f23c14d780 # v2.86.1 + uses: taiki-e/install-action@6cd13508893c0e7eab5f273c2575d3859bd7229a # v2.86.6 with: tool: cargo-nextest From 97fbe2de61220eb68e5bfc41cb9bdef3920cde73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:22:58 +0800 Subject: [PATCH 11/74] chore(deps): Bump the codeql-action group with 2 updates (#3114) Bumps the codeql-action group with 2 updates: [github/codeql-action/init](https://github.com/github/codeql-action) and [github/codeql-action/analyze](https://github.com/github/codeql-action). Updates `github/codeql-action/init` from 4.37.7 to 4.37.8 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd...db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28) Updates `github/codeql-action/analyze` from 4.37.7 to 4.37.8 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd...db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28) --- updated-dependencies: - dependency-name: github/codeql-action/init dependency-version: 4.37.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action - dependency-name: github/codeql-action/analyze dependency-version: 4.37.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 876a9b7116..9d9aba21d3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -46,11 +46,11 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 with: category: "/language:actions" From c4b43ad16cb64377acd585aed446cddedbc873f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:43:20 +0800 Subject: [PATCH 12/74] chore(deps): Bump uuid from 1.24.0 to 1.25.0 (#3113) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.24.0 to 1.25.0. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.24.0...1.25.0) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.25.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e06ed57b47..3a5c79a550 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7705,7 +7705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -8340,9 +8340,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ "getrandom 0.4.3", "js-sys", From e9443b9738fc31a2f98ede54e0e144ad0f54be94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:10:55 +0800 Subject: [PATCH 13/74] chore(deps-dev): Bump huggingface-hub from 1.27.0 to 1.28.0 in /bindings/python (#3112) chore(deps-dev): Bump huggingface-hub in /bindings/python Bumps [huggingface-hub](https://github.com/huggingface/huggingface_hub) from 1.27.0 to 1.28.0. - [Release notes](https://github.com/huggingface/huggingface_hub/releases) - [Commits](https://github.com/huggingface/huggingface_hub/compare/v1.27.0...v1.28.0) --- updated-dependencies: - dependency-name: huggingface-hub dependency-version: 1.28.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bindings/python/uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock index c052618d97..4676a7f63c 100644 --- a/bindings/python/uv.lock +++ b/bindings/python/uv.lock @@ -404,7 +404,7 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "1.27.0" +version = "1.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -417,9 +417,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3e/9b/ddf3d02a8681f1b9ce52fda03d755dad6b74c4f8172304c4c8d2975450f9/huggingface_hub-1.27.0.tar.gz", hash = "sha256:c1fed40ea82a6b41b477f5243546549b792ae0a93abcea608cff66089bf8f8df", size = 942668, upload-time = "2026-08-07T12:48:05.161Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/ae/222a91937ebee7f62c0ca8f5ee0afd97577caf24c0abb927d1f5c7e9f6d2/huggingface_hub-1.28.0.tar.gz", hash = "sha256:46a2e950c09234de54093d587d1675382f0d08dbd600d9fb599b5932f5b2c6cb", size = 959609, upload-time = "2026-08-18T12:27:15.101Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/d8/95b735e183957c1f26d94c52977f09d466d55119cbbc1558ea4975e4c216/huggingface_hub-1.27.0-py3-none-any.whl", hash = "sha256:7df6827c2f956c60fbaa64646e979e566db76f619dd0a9729dfb8c5a3eb4f68d", size = 784926, upload-time = "2026-08-07T12:48:02.905Z" }, + { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, ] [[package]] From d6d06f3c4a60e5f3a6d3d789a43513a919d9eacb Mon Sep 17 00:00:00 2001 From: Matt Faltyn <76570855+mattfaltyn@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:15:37 +0200 Subject: [PATCH 14/74] fix(spec): reject unequal map default arrays (#3110) --- crates/iceberg/src/spec/datatypes.rs | 61 +++++++++++++++++------ crates/iceberg/src/spec/values/literal.rs | 11 ++++ crates/iceberg/src/spec/values/tests.rs | 19 +++++++ 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/crates/iceberg/src/spec/datatypes.rs b/crates/iceberg/src/spec/datatypes.rs index 04c60fdc28..e53f856c00 100644 --- a/crates/iceberg/src/spec/datatypes.rs +++ b/crates/iceberg/src/spec/datatypes.rs @@ -19,7 +19,6 @@ * Data Types */ use std::collections::HashMap; -use std::convert::identity; use std::fmt; use std::ops::Index; use std::sync::{Arc, OnceLock}; @@ -554,7 +553,7 @@ impl fmt::Display for StructType { } #[derive(Debug, PartialEq, Serialize, Deserialize, Eq, Clone)] -#[serde(from = "SerdeNestedField", into = "SerdeNestedField")] +#[serde(try_from = "SerdeNestedField", into = "SerdeNestedField")] /// A struct is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema. /// Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type. /// Fields may have an optional comment or doc string. Fields can have default values. @@ -591,25 +590,30 @@ struct SerdeNestedField { pub write_default: Option, } -impl From for NestedField { - fn from(value: SerdeNestedField) -> Self { - NestedField { +impl TryFrom for NestedField { + type Error = crate::Error; + + fn try_from(value: SerdeNestedField) -> Result { + let initial_default = value + .initial_default + .map(|x| Literal::try_from_json(x, &value.field_type)) + .transpose()? + .flatten(); + let write_default = value + .write_default + .map(|x| Literal::try_from_json(x, &value.field_type)) + .transpose()? + .flatten(); + + Ok(NestedField { id: value.id, name: value.name, required: value.required, - initial_default: value.initial_default.and_then(|x| { - Literal::try_from_json(x, &value.field_type) - .ok() - .and_then(identity) - }), - write_default: value.write_default.and_then(|x| { - Literal::try_from_json(x, &value.field_type) - .ok() - .and_then(identity) - }), + initial_default, + write_default, field_type: value.field_type, doc: value.doc, - } + }) } } @@ -1354,6 +1358,31 @@ mod tests { assert_eq!(field, roundtrip); } + #[test] + fn nested_field_rejects_invalid_map_defaults() { + for default_name in ["initial-default", "write-default"] { + let json = format!( + r#"{{ + "id": 1, + "name": "properties", + "required": false, + "type": {{ + "type": "map", + "key-id": 2, + "key": "string", + "value-id": 3, + "value-required": false, + "value": "int" + }}, + "{default_name}": {{"keys": ["a", "b"], "values": [1]}} + }}"# + ); + + let error = serde_json::from_str::(&json).unwrap_err(); + assert!(error.to_string().contains("must have the same length")); + } + } + #[test] fn struct_type_with_type_field() { // Test that StructType properly deserializes JSON with "type":"struct" field diff --git a/crates/iceberg/src/spec/values/literal.rs b/crates/iceberg/src/spec/values/literal.rs index 5296eff2a2..194b1c3b94 100644 --- a/crates/iceberg/src/spec/values/literal.rs +++ b/crates/iceberg/src/spec/values/literal.rs @@ -598,6 +598,17 @@ impl Literal { if let (Some(JsonValue::Array(keys)), Some(JsonValue::Array(values))) = (object.remove("keys"), object.remove("values")) { + if keys.len() != values.len() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Map keys and values must have the same length, got {} keys and {} values", + keys.len(), + values.len() + ), + )); + } + Ok(Some(Literal::Map(Map::from_iter( keys.into_iter() .zip(values) diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index 5ee4ac8d02..8c72332f13 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -426,6 +426,25 @@ fn json_map() { ); } +#[test] +fn json_map_rejects_mismatched_key_value_lengths() { + let map_type = Type::Map(MapType { + key_field: NestedField::map_key_element(0, Primitive(PrimitiveType::String)).into(), + value_field: NestedField::map_value_element(1, Primitive(PrimitiveType::Int), true).into(), + }); + + for record in [ + r#"{"keys":["a","b"],"values":[1]}"#, + r#"{"keys":["a"],"values":[1,2]}"#, + ] { + let value = serde_json::from_str::(record).unwrap(); + let error = Literal::try_from_json(value, &map_type).unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.to_string().contains("must have the same length")); + } +} + #[test] fn avro_bytes_boolean() { let bytes = vec![1u8]; From 5c37021834c5391f499bb04e825e5b3d3282bdee Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 31 Aug 2026 08:51:26 -0700 Subject: [PATCH 15/74] Prune data columns for metadata-only projections (#3117) * Prune data columns for metadata-only projections A projection of only metadata columns (e.g. SELECT _row_id) reads no data columns for their values, since they are synthesized or constant. Without an independent row-count source, the projection mask fell back to reading every data column just to recover the row count. Install the RowNumber virtual column (derived from row-group metadata, no column I/O) as that source for any metadata-only projection, so the read is pruned to zero data columns. COUNT(*) (an empty projection) is left unchanged. Applies to all metadata-only projections: _row_id, _last_updated_sequence_number, _file, _spec_id, and _partition. Measured on a 58 MB, 200k-row, 30-column Parquet file, projecting only _last_updated_sequence_number: this change (pruned): 524 KB read, 6 ms before (read all columns): 58.6 MB read, 1289 ms * Incorporate PR feedback --- crates/iceberg/src/arrow/reader/pipeline.rs | 376 +++++++++++++++++--- 1 file changed, 317 insertions(+), 59 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 9b66bea4cc..27728242e1 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -102,6 +102,18 @@ impl ArrowReader { } } +// Metadata columns synthesized without reading any data column, so a projection of only +// these can be pruned to zero data columns. Narrower than `is_metadata_field`, which also +// matches `_deleted` -- excluded here because it has no synthesis handler. +const PRUNABLE_METADATA_FIELDS: &[i32] = &[ + RESERVED_FIELD_ID_FILE, + RESERVED_FIELD_ID_SPEC_ID, + RESERVED_FIELD_ID_PARTITION, + RESERVED_FIELD_ID_POS, + RESERVED_FIELD_ID_ROW_ID, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, +]; + /// Per-scan state for processing [`FileScanTask`]s. Created once per /// [`ArrowReader::read`] call and cloned per task. #[derive(Clone)] @@ -230,7 +242,15 @@ impl FileScanTaskReader { // column, so nothing is synthesized and the column is not needed. let need_row_number = project_pos || (project_row_id && task.first_row_id.is_some()); - let arrow_metadata = if need_row_number { + let field_ids = task.project_field_ids(); + let metadata_only_projection = !field_ids.is_empty() + && field_ids + .iter() + .all(|id| PRUNABLE_METADATA_FIELDS.contains(id)); + + let install_row_number = need_row_number || metadata_only_projection; + + let arrow_metadata = if install_row_number { let row_number_field = Arc::new( Field::new(RESERVED_COL_NAME_POS, DataType::Int64, false) .with_metadata(HashMap::from([( @@ -349,24 +369,15 @@ impl FileScanTaskReader { // A metadata-only projection leaves `project_field_ids_without_metadata` empty, // which `get_arrow_projection_mask` maps to "read all columns" (so `COUNT(*)` still - // gets a row count). Downgrade that to "read no data columns" when a row-count - // source exists independently of the data columns: the RowNumber virtual column - // (installed above under `need_row_number`, which covers `_pos` and `_row_id` - // synthesis) or a physical `_last_updated_sequence_number` leaf unioned in below - // (that column does not install RowNumber, so it is a separate source). Pure-constant - // / `COUNT(*)` projections have neither and must keep reading all columns to preserve - // the row count. + // gets a row count). Downgrade that to "read no data columns": `install_row_number` + // put the RowNumber virtual column on every metadata-only projection as a row-count + // source independent of the data columns, so the count survives with zero data + // columns read. `COUNT(*)` (an empty projection) has no RowNumber and keeps reading + // all columns to preserve the row count. // - // This runs BEFORE the union so the physical leaves are added onto a `none` base, - // pruning the read to just those leaves (`union` with an `all` base stays `all`). - // - // Known follow-up: a `_row_id`-only (or `_last_updated_sequence_number`-only) - // projection with a null `first_row_id` has no row-count source here, so it still - // reads all columns just to emit an all-null column. Pruning that to a minimal - // row-count source is left as a future optimization. - if project_field_ids_without_metadata.is_empty() - && (need_row_number || coalesce_last_updated_seq_leaf.is_some()) - { + // This runs BEFORE the union so any physical metadata leaf is added onto a `none` + // base, pruning the read to just that leaf (`union` with an `all` base stays `all`). + if project_field_ids_without_metadata.is_empty() && install_row_number { projection_mask = ProjectionMask::none(record_batch_stream_builder.parquet_schema().num_columns()); } @@ -1666,6 +1677,163 @@ mod tests { ); } + #[tokio::test] + async fn test_row_id_only_null_first_row_id_reads_no_data_columns() { + // A null `first_row_id` (v1/v2, or a pre-upgrade v3 snapshot) nulls the whole + // `_row_id` column, so nothing is synthesized -- but the column still has a length. + // The RowNumber counter installed for the metadata-only projection supplies it, so + // the scan reads no data columns instead of reading everything just for the count. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let mut meta_only = metadata_projection_task( + write_parquet_with_wide_column(dir, "row_id_null.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![RESERVED_FIELD_ID_ROW_ID], + ); + meta_only.first_row_id = None; + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + assert_row_id_column(&batches, &[None, None, None]); + + let mut with_data = metadata_projection_task( + write_parquet_with_wide_column(dir, "row_id_null_ref.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![2, RESERVED_FIELD_ID_ROW_ID], + ); + with_data.first_row_id = None; + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_row_id-only scan with a null first_row_id should read fewer bytes than a scan \ + of the wide column: {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_last_updated_seq_only_reads_no_data_columns() { + use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER; + + // `SELECT _last_updated_sequence_number` derives a per-row constant (the data + // sequence number) with no physical column to read. The metadata-only RowNumber + // counter supplies the row count, so the scan prunes the data columns rather than + // reading them all just to size the constant. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let lusn_only_task = |file_path: String, project_field_ids: Vec| { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(id_and_wide_schema()) + .with_project_field_ids(project_field_ids) + .with_first_row_id(Some(42)) + .with_data_sequence_number(Some(7)) + .with_case_sensitive(false) + .build() + }; + + let meta_only = lusn_only_task( + write_parquet_with_wide_column(dir, "lusn_only.parquet", vec![], vec![]), + vec![RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER], + ); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + assert_last_updated_seq_column(&batches, &[Some(7), Some(7), Some(7)]); + + let with_data = lusn_only_task( + write_parquet_with_wide_column(dir, "lusn_only_ref.parquet", vec![], vec![]), + vec![2, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_last_updated_sequence_number-only scan should read fewer bytes than a scan of \ + the wide column: {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_partition_only_reads_no_data_columns() { + use arrow_array::StructArray; + + use crate::metadata_columns::{RESERVED_COL_NAME_PARTITION, RESERVED_FIELD_ID_PARTITION}; + use crate::spec::{Literal, PartitionSpec, Struct, Transform}; + + // `SELECT _partition` materializes a struct constant from the task's partition + // metadata, with no physical column to read. It is the only struct-constant metadata + // column; the RowNumber counter sizes it, so the scan prunes the data columns. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let schema = id_and_wide_schema(); + let spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .with_spec_id(7) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let unified_type = Arc::new(spec.partition_type(&schema).unwrap()); + let partition_data = Struct::from_iter(vec![Some(Literal::int(42))]); + + let partition_task = |file_path: String, project_field_ids: Vec| { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids) + .with_partition_spec(Some(spec.clone())) + .with_partition(Some(partition_data.clone())) + .with_unified_partition_type(Some(unified_type.clone())) + .with_case_sensitive(false) + .build() + }; + + let meta_only = partition_task( + write_parquet_with_wide_column(dir, "partition_only.parquet", vec![], vec![]), + vec![RESERVED_FIELD_ID_PARTITION], + ); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + let partition_col = batches[0] + .column_by_name(RESERVED_COL_NAME_PARTITION) + .expect("_partition column should be present") + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(partition_col.len(), 3); + let inner = partition_col + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(inner.values(), &[42, 42, 42]); + + let with_data = partition_task( + write_parquet_with_wide_column(dir, "partition_only_ref.parquet", vec![], vec![]), + vec![2, RESERVED_FIELD_ID_PARTITION], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_partition-only scan should read fewer bytes than a scan of the wide column: \ + {meta_only_bytes} vs {with_data_bytes}" + ); + } + #[tokio::test] async fn test_row_id_resolves_alongside_id_less_leaf() { // A file with an id-less leaf (mimicking a Variant column's internal metadata/value @@ -3179,9 +3347,9 @@ mod tests { RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, }; - // `_last_updated_sequence_number` alone (no `_pos`, no data column). The physical - // leaf is the sole row source, so it -- not the RowNumber virtual column -- must - // drive the `none()` downgrade and prune the read to just that leaf. + // `_last_updated_sequence_number` alone (no `_pos`, no data column). This is a + // metadata-only projection, so RowNumber drives the `none()` downgrade and the + // physical leaf is unioned back in, pruning the read to just that leaf. let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path().to_str().unwrap(); let write = |name: &str| { @@ -3238,7 +3406,7 @@ mod tests { } #[tokio::test] - async fn test_seq_only_projection_null_first_row_id_preserves_row_count() { + async fn test_seq_only_null_first_row_id_reads_no_data_columns() { use crate::metadata_columns::{ RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, @@ -3246,30 +3414,37 @@ mod tests { // Seq-only projection with a null first_row_id: the column is nulled and the // physical leaf is NOT read (the gated `coalesce_last_updated_seq_leaf` is None). - // The downgrade must therefore not fire -- keying off the raw `project_*` flag - // instead would drop the only readable column and lose the row count. + // This is a metadata-only projection, so RowNumber supplies the row count and the + // data columns are pruned; the row count is still 3 and the values all null. let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path().to_str().unwrap(); - let file_path = write_parquet_with_wide_column( - dir, - "seq_only_null_first.parquet", - vec![physical_last_updated_seq_field()], - vec![Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef], - ); - let task = FileScanTask::builder() - .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) - .with_start(0) - .with_length(0) - .with_data_file_path(file_path) - .with_data_file_format(DataFileFormat::Parquet) - .with_schema(id_and_wide_schema()) - .with_project_field_ids(vec![RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER]) - .with_first_row_id(None) - .with_data_sequence_number(Some(9)) - .with_case_sensitive(false) - .build(); - let (batches, _) = scan_task(task).await; + let seq_task = |name: &str, ids: Vec| { + let file_path = write_parquet_with_wide_column( + dir, + name, + vec![physical_last_updated_seq_field()], + vec![Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef], + ); + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(id_and_wide_schema()) + .with_project_field_ids(ids) + .with_first_row_id(None) + .with_data_sequence_number(Some(9)) + .with_case_sensitive(false) + .build() + }; + + let meta_only = seq_task("seq_only_null_first.parquet", vec![ + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + assert_eq!(batches[0].num_columns(), 1); let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 3); let seq_col = batches[0] @@ -3278,30 +3453,37 @@ mod tests { let seq_col = cast(seq_col, &DataType::Int64).unwrap(); let seq_col = seq_col.as_any().downcast_ref::().unwrap(); assert!((0..3).all(|i| seq_col.is_null(i))); + + let with_data = seq_task("seq_only_null_first_ref.parquet", vec![ + 2, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "seq-only scan with a null first_row_id should read fewer bytes than a scan of \ + the wide column: {meta_only_bytes} vs {with_data_bytes}" + ); } #[tokio::test] - async fn test_file_only_projection_preserves_row_count() { + async fn test_file_only_reads_no_data_columns() { use crate::metadata_columns::RESERVED_COL_NAME_FILE; + // `SELECT _file` materializes a per-file string constant with no physical column to + // read. The metadata-only RowNumber counter sizes it, so the scan prunes the data + // columns rather than reading them all just to recover the row count. let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path().to_str().unwrap(); - let file_path = write_plain_parquet(dir, "file_only.parquet", vec![], vec![]); - let schema = Arc::new( - Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - ]) - .build() - .unwrap(), - ); - let task = - metadata_projection_task(file_path.clone(), schema, vec![RESERVED_FIELD_ID_FILE]); - let (batches, _) = scan_task(task).await; - // A pure-constant projection has no independent row source, so the row count must - // still come from the file (the `empty -> all()` path is preserved for this case). + let file_path = write_parquet_with_wide_column(dir, "file_only.parquet", vec![], vec![]); + let meta_only = metadata_projection_task(file_path.clone(), id_and_wide_schema(), vec![ + RESERVED_FIELD_ID_FILE, + ]); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 3); let file_col = batches[0] @@ -3310,6 +3492,82 @@ mod tests { let file_col = cast(file_col, &DataType::Utf8).unwrap(); let file_col = file_col.as_any().downcast_ref::().unwrap(); assert_eq!(file_col.value(0), file_path); + + let with_data = metadata_projection_task( + write_parquet_with_wide_column(dir, "file_only_ref.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![2, RESERVED_FIELD_ID_FILE], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_file-only scan should read fewer bytes than a scan of the wide column: \ + {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_spec_id_only_reads_no_data_columns() { + use crate::metadata_columns::{RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_SPEC_ID}; + use crate::spec::{PartitionSpec, Transform}; + + // `SELECT _spec_id` materializes an int constant from the task's partition spec id, + // with no physical column to read. The RowNumber counter sizes it, so the scan + // prunes the data columns rather than reading them all to recover the row count. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let schema = id_and_wide_schema(); + let spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .with_spec_id(7) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + let spec_id_task = |file_path: String, project_field_ids: Vec| { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids) + .with_partition_spec(Some(spec.clone())) + .with_case_sensitive(false) + .build() + }; + + let meta_only = spec_id_task( + write_parquet_with_wide_column(dir, "spec_id_only.parquet", vec![], vec![]), + vec![RESERVED_FIELD_ID_SPEC_ID], + ); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 3); + let spec_id_col = batches[0] + .column_by_name(RESERVED_COL_NAME_SPEC_ID) + .expect("_spec_id column should be present"); + let spec_id_col = cast(spec_id_col, &DataType::Int32).unwrap(); + let spec_id_col = spec_id_col.as_any().downcast_ref::().unwrap(); + assert_eq!(spec_id_col.values(), &[7, 7, 7]); + + let with_data = spec_id_task( + write_parquet_with_wide_column(dir, "spec_id_only_ref.parquet", vec![], vec![]), + vec![2, RESERVED_FIELD_ID_SPEC_ID], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_spec_id-only scan should read fewer bytes than a scan of the wide column: \ + {meta_only_bytes} vs {with_data_bytes}" + ); } #[tokio::test] From 6d65786d7a4e19d6e62ac6b5df79e88841eae7be Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:29:00 +0800 Subject: [PATCH 16/74] refactor(memory): derive catalog configuration from properties (#3101) * refactor(memory): derive catalog configuration from properties Replace manual catalog property extraction with the shared derive while preserving warehouse validation and forwarding unmodeled properties. Tracks [apache/iceberg-rust#3095](https://github.com/apache/iceberg-rust/issues/3095). Generated-by: Codex * refactor(memory): retain typed catalog properties Remove the redundant config wrapper and catch-all property field, expose the typed properties, and retain the catalog name and typed properties on MemoryCatalog. Addresses review feedback on [apache/iceberg-rust#3101](https://github.com/apache/iceberg-rust/pull/3101). Generated-by: Codex * refactor(memory): retain raw catalog properties * fix(memory): defer missing warehouse validation * refactor(memory): move raw properties into file io * fix(memory): restore catalog validation contract * refactor(memory): retain public catalog properties * refactor(memory): forward raw catalog properties --- crates/iceberg/public-api.txt | 2 +- crates/iceberg/src/catalog/memory/catalog.rs | 185 ++++++++++++------- 2 files changed, 120 insertions(+), 67 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 4aeddb665a..d61912c906 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1112,7 +1112,7 @@ pub fn iceberg::memory::MemoryCatalog::update_namespace<'life0, 'life1, 'async_t pub fn iceberg::memory::MemoryCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct iceberg::memory::MemoryCatalogBuilder impl core::default::Default for iceberg::memory::MemoryCatalogBuilder -pub fn iceberg::memory::MemoryCatalogBuilder::default() -> Self +pub fn iceberg::memory::MemoryCatalogBuilder::default() -> iceberg::memory::MemoryCatalogBuilder impl core::fmt::Debug for iceberg::memory::MemoryCatalogBuilder pub fn iceberg::memory::MemoryCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::CatalogBuilder for iceberg::memory::MemoryCatalogBuilder diff --git a/crates/iceberg/src/catalog/memory/catalog.rs b/crates/iceberg/src/catalog/memory/catalog.rs index 85fcd824ef..aae0f7d5ac 100644 --- a/crates/iceberg/src/catalog/memory/catalog.rs +++ b/crates/iceberg/src/catalog/memory/catalog.rs @@ -23,6 +23,7 @@ use std::sync::Arc; use async_trait::async_trait; use futures::lock::{Mutex, MutexGuard}; +use iceberg_property_macro::Properties; use itertools::Itertools; use super::namespace_state::NamespaceState; @@ -43,29 +44,13 @@ pub const MEMORY_CATALOG_WAREHOUSE: &str = "warehouse"; const LOCATION: &str = "location"; /// Builder for [`MemoryCatalog`]. -#[derive(Debug)] +#[derive(Debug, Default)] pub struct MemoryCatalogBuilder { - config: MemoryCatalogConfig, storage_factory: Option>, kms_client_factory: Option>, runtime: Option, } -impl Default for MemoryCatalogBuilder { - fn default() -> Self { - Self { - config: MemoryCatalogConfig { - name: None, - warehouse: "".to_string(), - props: HashMap::new(), - }, - storage_factory: None, - kms_client_factory: None, - runtime: None, - } - } -} - impl CatalogBuilder for MemoryCatalogBuilder { type C = MemoryCatalog; @@ -85,69 +70,70 @@ impl CatalogBuilder for MemoryCatalogBuilder { } fn load( - mut self, + self, name: impl Into, props: HashMap, ) -> impl Future> + Send { - self.config.name = Some(name.into()); - - if props.contains_key(MEMORY_CATALOG_WAREHOUSE) { - self.config.warehouse = props - .get(MEMORY_CATALOG_WAREHOUSE) - .cloned() - .unwrap_or_default() - } - - // Collect other remaining properties - self.config.props = props - .into_iter() - .filter(|(k, _)| k != MEMORY_CATALOG_WAREHOUSE) - .collect(); + let name = name.into(); async move { - if self.config.name.is_none() { - Err(Error::new( - ErrorKind::DataInvalid, - "Catalog name is required", - )) - } else if self.config.warehouse.is_empty() { - Err(Error::new( + let catalog_properties = MemoryCatalogProperties::from_properties(&props)?; + if catalog_properties.warehouse.is_empty() { + return Err(Error::new( ErrorKind::DataInvalid, "Catalog warehouse is required", - )) - } else { - let runtime = self.runtime.unwrap_or_else(Runtime::current); - let kms_client = match self.kms_client_factory { - Some(factory) => Some(factory.create_kms_client(&self.config.props).await?), - None => None, - }; - MemoryCatalog::new(self.config, self.storage_factory, runtime, kms_client) + )); } + + let runtime = self.runtime.unwrap_or_else(Runtime::current); + let kms_client = match self.kms_client_factory { + Some(factory) => Some(factory.create_kms_client(&props).await?), + None => None, + }; + MemoryCatalog::new( + name, + catalog_properties, + props, + self.storage_factory, + runtime, + kms_client, + ) } } } -#[derive(Clone, Debug)] -pub(crate) struct MemoryCatalogConfig { - name: Option, +/// Memory catalog properties parsed from a catalog property map. +#[derive(Debug, Properties)] +pub(crate) struct MemoryCatalogProperties { + #[property(key = MEMORY_CATALOG_WAREHOUSE, default = "")] warehouse: String, - props: HashMap, } /// Memory catalog implementation. -#[derive(Debug)] pub struct MemoryCatalog { + name: String, + properties: MemoryCatalogProperties, root_namespace_state: Mutex, file_io: FileIO, - warehouse_location: String, runtime: Runtime, kms_client: Option>, } +impl std::fmt::Debug for MemoryCatalog { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryCatalog") + .field("name", &self.name) + .field("properties", &self.properties) + .finish_non_exhaustive() + } +} + impl MemoryCatalog { /// Creates a memory catalog. fn new( - config: MemoryCatalogConfig, + name: String, + properties: MemoryCatalogProperties, + props: HashMap, storage_factory: Option>, runtime: Runtime, kms_client: Option>, @@ -156,9 +142,10 @@ impl MemoryCatalog { let factory = storage_factory.unwrap_or_else(|| Arc::new(MemoryStorageFactory)); Ok(Self { + name, + properties, + file_io: FileIOBuilder::new(factory).with_props(props).build(), root_namespace_state: Mutex::new(NamespaceState::default()), - file_io: FileIOBuilder::new(factory).with_props(config.props).build(), - warehouse_location: config.warehouse, runtime, kms_client, }) @@ -308,7 +295,11 @@ impl Catalog for MemoryCatalog { let namespace_properties = root_namespace_state.get_properties(namespace_ident)?; let location_prefix = match namespace_properties.get(LOCATION) { Some(namespace_location) => namespace_location.clone(), - None => format!("{}/{}", self.warehouse_location, namespace_ident.join("/")), + None => format!( + "{}/{}", + self.properties.warehouse, + namespace_ident.join("/") + ), }; let location = format!("{}/{}", location_prefix, table_ident.name()); @@ -458,6 +449,57 @@ pub(crate) mod tests { temp_dir.path().to_str().unwrap().to_string() } + #[test] + fn test_catalog_properties() { + let properties = MemoryCatalogProperties::from_properties(&HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )])) + .unwrap(); + + assert_eq!(properties.warehouse, "memory:///warehouse"); + } + + #[test] + fn test_catalog_properties_warehouse_defaults_to_empty() { + let missing = MemoryCatalogProperties::from_properties(&HashMap::new()).unwrap(); + let explicitly_empty = MemoryCatalogProperties::from_properties(&HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + String::new(), + )])) + .unwrap(); + + assert_eq!(missing.warehouse, ""); + assert_eq!(explicitly_empty.warehouse, ""); + } + + #[tokio::test] + async fn test_catalog_forwards_properties_to_file_io() { + let catalog = MemoryCatalogBuilder::default() + .load( + "memory", + HashMap::from([ + ( + MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + ), + ("custom.property".to_string(), "value".to_string()), + ]), + ) + .await + .unwrap(); + let file_io_props = catalog.file_io.config().props(); + + assert_eq!( + file_io_props.get("custom.property"), + Some(&"value".to_string()) + ); + assert_eq!( + file_io_props.get(MEMORY_CATALOG_WAREHOUSE), + Some(&"memory:///warehouse".to_string()) + ); + } + pub(crate) async fn new_memory_catalog() -> impl Catalog { let warehouse_location = temp_path(); MemoryCatalogBuilder::default() @@ -1401,17 +1443,28 @@ pub(crate) mod tests { } #[tokio::test] - async fn test_create_table_throws_error_if_table_location_and_namespace_location_and_warehouse_location_are_missing() - { - let catalog = MemoryCatalogBuilder::default() + async fn test_load_throws_error_if_warehouse_is_missing() { + let error = MemoryCatalogBuilder::default() .load("memory", HashMap::from([])) - .await; + .await + .unwrap_err(); - assert!(catalog.is_err()); - assert_eq!( - catalog.unwrap_err().to_string(), - "DataInvalid => Catalog warehouse is required" - ); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!(error.message(), "Catalog warehouse is required"); + } + + #[tokio::test] + async fn test_load_throws_error_if_warehouse_is_empty() { + let error = MemoryCatalogBuilder::default() + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), String::new())]), + ) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!(error.message(), "Catalog warehouse is required"); } #[tokio::test] From 1c27c683e7771e67badc7c6b3686958ca75664ce Mon Sep 17 00:00:00 2001 From: Daniel Carl Jones Date: Wed, 2 Sep 2026 10:51:29 +0100 Subject: [PATCH 17/74] chore: Merge changes from 0.10.x to main (#3074) * chore: Add LICENSE and NOTICE files for crates missing them in 0.10.0 (#2830) ## Which issue does this PR close? For release: #2527 ## What changes are included in this PR? These changes are cherry-picks of #2823 and #2828. The crates were missing the LICENSE and NOTICE files required for the distributed source code. ## Are these changes tested? Tested using the following script provided by @kevinjqliu. Introducing this in CI is left as a follow-up (https://github.com/apache/iceberg-rust/issues/2827). ```bash set -Eeuo pipefail failed=0 while IFS= read -r crate; do echo "Checking ${crate}..." files="$(cargo package \ --package "${crate}" \ --list \ --locked)" for required in LICENSE NOTICE; do if ! grep -Fxq "${required}" <<<"${files}"; then echo "::error::${crate} package is missing top-level ${required}" failed=1 fi done done < <( cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | select(.publish != []) | .name' ) exit "${failed}" ``` --------- Co-authored-by: Andrew Kane * chore: Update changelog for 0.10.0 RC4 (#2831) ## Which issue does this PR close? Follow up for 0.10.0 RC4. ## What changes are included in this PR? Change log entries for missing changes in 0.10.0 RC4. The changelog is worded for the original commits to `main`, and not as the squashed cherry pick in #2830. ## Are these changes tested? N/A * chore: Update crossbeam-epoch to 0.9.20, pin CI ruff version to 0.15.22 (#2911) ## Which issue does this PR close? Small changes needed for 0.10.1. ## What changes are included in this PR? First, address advisory in crossbeam-epoch with patch version update. Second, to address CI, pin Ruff linter for Python to version available when iceberg-rust 0.10.0 was released. ## Are these changes tested? Already covered by CI. * chore: Bump version to 0.10.1, update changelog (#2909) ## Which issue does this PR close? This bumps the crate and pyiceberg-core version to 0.10.1, for the 0.10.1 release. Additionally, it pins ruff to 0.15.22 which was the version used for iceberg-rust 0.10.0. ## What changes are included in this PR? Version bump, updated dependency TSV manifests, updated changelog. Pins CI version of ruff Python linter. ## Are these changes tested? N/A --------- Co-authored-by: Andrew Kane --- .github/workflows/bindings_python_ci.yml | 2 +- CHANGELOG.md | 7 +++++ Cargo.lock | 34 ++++++++++++------------ Cargo.toml | 2 +- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index f711f038d3..a747c98842 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -56,7 +56,7 @@ jobs: enable-cache: true - name: Install tools run: | - uv tool install ruff + uv tool install ruff@0.15.22 - name: Check format working-directory: "bindings/python" run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c43de2833..1e06bafa91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). +## [v0.10.1] - 2026-07-28 + +* chore: Update crossbeam-epoch to 0.9.20, pin CI ruff version to 0.15.22 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2911 + ## [v0.10.0] - 2026-07-07 ### Breaking Changes @@ -288,6 +292,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). * Fix zizmor pin by @blackmwk in https://github.com/apache/iceberg-rust/pull/2762 * feat(encryption) [8/N] Read encrypted manifest file by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2586 * Reject negative positional delete positions by @fallintoplace in https://github.com/apache/iceberg-rust/pull/2631 +* chore: Update changelog for 0.10.0 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2782 +* Include license and notice files in iceberg-storage-opendal crate by @ankane in https://github.com/apache/iceberg-rust/pull/2823 +* fix(license): Add missing LICENSE and NOTICE symlinks for iceberg-cache-moka and iceberg-catalog-loader by @dannycjones in https://github.com/apache/iceberg-rust/pull/2828 ## [v0.9.0] - 2026-03-10 diff --git a/Cargo.lock b/Cargo.lock index 3a5c79a550..326457640c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3759,7 +3759,7 @@ dependencies = [ [[package]] name = "iceberg" -version = "0.10.0" +version = "0.10.1" dependencies = [ "aes-gcm", "anyhow", @@ -3824,7 +3824,7 @@ dependencies = [ [[package]] name = "iceberg-cache-moka" -version = "0.10.0" +version = "0.10.1" dependencies = [ "iceberg", "moka", @@ -3832,7 +3832,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-glue" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "async-trait", @@ -3847,7 +3847,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-hms" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "async-trait", @@ -3870,7 +3870,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-loader" -version = "0.10.0" +version = "0.10.1" dependencies = [ "async-trait", "iceberg", @@ -3890,7 +3890,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" -version = "0.10.0" +version = "0.10.1" dependencies = [ "async-trait", "bytes", @@ -3913,7 +3913,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-s3tables" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "arrow-array", @@ -3933,7 +3933,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-sql" -version = "0.10.0" +version = "0.10.1" dependencies = [ "async-trait", "iceberg", @@ -3948,7 +3948,7 @@ dependencies = [ [[package]] name = "iceberg-datafusion" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "async-trait", @@ -3965,7 +3965,7 @@ dependencies = [ [[package]] name = "iceberg-examples" -version = "0.10.0" +version = "0.10.1" dependencies = [ "futures", "iceberg", @@ -3976,7 +3976,7 @@ dependencies = [ [[package]] name = "iceberg-integration-tests" -version = "0.10.0" +version = "0.10.1" dependencies = [ "arrow-array", "arrow-schema", @@ -3993,7 +3993,7 @@ dependencies = [ [[package]] name = "iceberg-playground" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "clap", @@ -4015,7 +4015,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" -version = "0.10.0" +version = "0.10.1" dependencies = [ "iceberg", "proc-macro2", @@ -4028,7 +4028,7 @@ dependencies = [ [[package]] name = "iceberg-sqllogictest" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "async-trait", @@ -4050,7 +4050,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "async-trait", @@ -4071,7 +4071,7 @@ dependencies = [ [[package]] name = "iceberg_test_utils" -version = "0.10.0" +version = "0.10.1" dependencies = [ "iceberg", "tracing-subscriber", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "pyiceberg_core_rust" -version = "0.10.0" +version = "0.10.1" dependencies = [ "arrow", "datafusion-ffi", diff --git a/Cargo.toml b/Cargo.toml index cea9ede78a..31643aeb0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ resolver = "2" edition = "2024" homepage = "https://rust.iceberg.apache.org/" publish = false -version = "0.10.0" +version = "0.10.1" license = "Apache-2.0" repository = "https://github.com/apache/iceberg-rust" From 34ad7e0a52e119fc6032831ac587ff3026e8d34d Mon Sep 17 00:00:00 2001 From: Daniel Carl Jones Date: Wed, 2 Sep 2026 11:03:13 +0100 Subject: [PATCH 18/74] chore(ci): Fail CI when a publishable crate is missing LICENSE or NOTICE (#3084) * chore(ci): Fail CI when a publishable crate is missing LICENSE or NOTICE Missing LICENSE and NOTICE files for newly introduced crates (#2823) were only discovered manually during release preparation, because nothing asserted their presence. Add dev/check_license_notice.sh, which lists the packaged files of every publishable crate with `cargo package --list` and fails when a top-level LICENSE or NOTICE is absent. Checking the packaged file list rather than the crate directory also catches files present on disk but excluded from the published crate. Because `cargo package --list` reports an entry even when a symlinked target is gone, the file contents are checked separately so a removed or emptied root copy fails too. Each crate's copy must also resolve to the repository root file itself, so the license text cannot drift from one crate to the next. Run it from the CI lint job so pull requests fail when a crate is missing either file, and stop excluding the root LICENSE and NOTICE from the pull_request path filter, since every crate symlinks to those copies and a PR removing them must still run the job. Also wire it into `make check` so it can be run locally the same way as the other lint checks. The check does not pass at this commit. It reports iceberg-property-macro, which is published but ships neither file, and iceberg-catalog-s3tables, whose LICENSE and NOTICE are real copies rather than the repository root files. Both gaps are fixed in the two commits that follow. Closes #2827 * chore: Add missing LICENSE and NOTICE to iceberg-property-macro iceberg-property-macro is published to crates.io but shipped neither a LICENSE nor a NOTICE, so the crate was distributed without the files an Apache release requires. It was missed because the crate was added after #1601, which introduced the files for the crates publishable at the time. Add the same relative symlinks to the repository root copies that every other publishable crate uses, which is what the new check in the preceding commit reports. * chore: Symlink s3tables LICENSE and NOTICE to the repository root Every publishable crate references the root LICENSE and NOTICE through a relative symlink. That convention was established in #1601, which added the files to the crates that were publishable at the time. `iceberg-catalog-s3tables` was `publish = false` back then so it was skipped, and #1916 later added real copies when the crate was made publishable. Every crate added since has followed the symlink convention, leaving s3tables as the only outlier. Replace the copies with symlinks so the root files remain the single source of truth and per-crate copies cannot drift from them. This is also required for the check added earlier in this branch to pass, which rejects a per-crate copy that is not the root file itself: the s3tables copies are byte-identical to the root files today, but nothing kept them that way. `cargo package` dereferences the symlinks, so the published crate is unchanged: LICENSE and NOTICE are still regular files with byte-identical content. * Update based on PR feedback --- .github/workflows/ci.yml | 11 +- Makefile | 5 +- crates/catalog/s3tables/LICENSE | 202 +------------------------------- crates/catalog/s3tables/NOTICE | 6 +- crates/property-macro/LICENSE | 1 + crates/property-macro/NOTICE | 1 + dev/check_license_notice.sh | 188 +++++++++++++++++++++++++++++ 7 files changed, 203 insertions(+), 211 deletions(-) mode change 100644 => 120000 crates/catalog/s3tables/LICENSE mode change 100644 => 120000 crates/catalog/s3tables/NOTICE create mode 120000 crates/property-macro/LICENSE create mode 120000 crates/property-macro/NOTICE create mode 100755 dev/check_license_notice.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ef51475aa..f25e737911 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,10 @@ on: - main pull_request: paths: - - '**' # Include all files and directories in the repository by default. - - '!.github/ISSUE_TEMPLATE/**' # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation. + # Artifacts that must be included in releases, such as LICENSE and NOTICE, are deliberately not excluded so we can verify they will be bundled. + # Include all files and directories, then filter out files and directories that don't impact code, tests, or release artifacts. + - '**' + - '!.github/ISSUE_TEMPLATE/**' - '!dev/release/**' - '!website/**' - '!.asf.yml' @@ -32,8 +34,6 @@ on: - '!.gitignore' - '!CONTRIBUTING.md' - '!CHANGELOG.md' - - '!LICENSE' - - '!NOTICE' - '!README.md' concurrency: @@ -75,6 +75,9 @@ jobs: - name: Check Cargo.lock is up-to-date run: cargo update --workspace --locked + - name: Check crates package LICENSE and NOTICE + run: make check-license-notice + - name: Cargo Machete run: cargo machete diff --git a/Makefile b/Makefile index 003efbfd15..d27893390a 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,10 @@ check-public-api: install-cargo-public-api done; \ if [ $$fail -ne 0 ]; then exit 1; fi -check: check-fmt check-clippy check-toml cargo-machete +check-license-notice: + bash ./dev/check_license_notice.sh + +check: check-fmt check-clippy check-toml cargo-machete check-license-notice doc-test: cargo test --no-fail-fast --doc --all-features --workspace diff --git a/crates/catalog/s3tables/LICENSE b/crates/catalog/s3tables/LICENSE deleted file mode 100644 index 261eeb9e9f..0000000000 --- a/crates/catalog/s3tables/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/crates/catalog/s3tables/LICENSE b/crates/catalog/s3tables/LICENSE new file mode 120000 index 0000000000..5853aaea53 --- /dev/null +++ b/crates/catalog/s3tables/LICENSE @@ -0,0 +1 @@ +../../../LICENSE \ No newline at end of file diff --git a/crates/catalog/s3tables/NOTICE b/crates/catalog/s3tables/NOTICE deleted file mode 100644 index 73b77516fa..0000000000 --- a/crates/catalog/s3tables/NOTICE +++ /dev/null @@ -1,5 +0,0 @@ -Apache Iceberg Rust -Copyright 2023-2026 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/crates/catalog/s3tables/NOTICE b/crates/catalog/s3tables/NOTICE new file mode 120000 index 0000000000..295f6bdb3a --- /dev/null +++ b/crates/catalog/s3tables/NOTICE @@ -0,0 +1 @@ +../../../NOTICE \ No newline at end of file diff --git a/crates/property-macro/LICENSE b/crates/property-macro/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/crates/property-macro/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/property-macro/NOTICE b/crates/property-macro/NOTICE new file mode 120000 index 0000000000..fb376cfaa4 --- /dev/null +++ b/crates/property-macro/NOTICE @@ -0,0 +1 @@ +../../NOTICE \ No newline at end of file diff --git a/dev/check_license_notice.sh b/dev/check_license_notice.sh new file mode 100755 index 0000000000..17757fb9aa --- /dev/null +++ b/dev/check_license_notice.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Every distributed artifact must ship its own LICENSE and NOTICE. +# Checking the packaged file list, rather than the crate directory, +# also catches files that exist on disk but are excluded from the published +# crate. Each crate's copy must be the repository root file itself, so the +# license text cannot drift from one crate to the next. + +set -Eeuo pipefail + +REQUIRED_FILES=(LICENSE NOTICE) + +usage() { + cat </dev/null 2>&1; then + echo "This script requires '${command_name}', but it is not installed." >&2 + return 1 + fi +} + +report_error() { + # GitHub Actions turns `::error::` lines into annotations on the job summary. + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + echo "::error::$1" + else + echo "ERROR: $1" >&2 + fi +} + +# Return name and manifest path for every crate that would be published in a release. +publishable_crates() { + # `publish` is an empty list for `publish = false` crates and null when the + # crate may be published to any registry. + cargo metadata --no-deps --format-version 1 | + jq -r '.packages[] | select(.publish != []) | "\(.name)\t\(.manifest_path)"' +} + +# Check the path resolves to a non-empty regular file, following any symlinks. +check_file_exists() { + local path="$1" + + # `-f` follows symlinks and rejects directories. + # `-s` additionally requires content, so an emptied file fails too. + if [ ! -f "${path}" ] || [ ! -s "${path}" ]; then + return 1 + fi +} + +# Return true if both paths are the same file. +is_same_file() { + # `-ef` is short for "equal file": it is true when both arguments resolve to the + # same device and inode. + [ "$1" -ef "$2" ] +} + +# Return true if the given path (second arg) is present in the list of paths (first arg). +packages_file_at_root() { + local packaged_paths="$1" + local file_name="$2" + + local packaged_path + + while IFS= read -r packaged_path; do + if [ "${packaged_path}" = "${file_name}" ]; then + return 0 + fi + done <<<"${packaged_paths}" + + return 1 +} + +# Verify that the necessary files are bundled in the crate. +# Accepts both crate name and its directory, so that artifacts can be fully verified including symlink resolution. +check_crate() { + local crate="$1" + local crate_dir="$2" + + local packaged_paths status repo_path + status=0 + + echo "Checking ${crate}..." + # --no-verify is no-op at the time of writing, but to be clear: packages will not be built, only expected files enumerated + # --allow-dirty allows for `make check` during development + if ! packaged_paths="$(cargo package --package "${crate}" --list --locked --no-verify --allow-dirty)"; then + report_error "${crate} could not be listed by cargo; see the error above" + return 1 + fi + + for required_file in "${REQUIRED_FILES[@]}"; do + # Not only verify that the path is included, but verify that it links to the root file. + repo_path="${crate_dir}/${required_file}" + if ! packages_file_at_root "${packaged_paths}" "${required_file}"; then + report_error "${crate} crate is missing ${required_file} in packaged file list" + status=1 + elif ! check_file_exists "${repo_path}"; then + report_error "${crate} ${required_file} does not resolve to a non-empty file: ${repo_path}" + status=1 + elif ! is_same_file "${repo_path}" "${REPO_ROOT}/${required_file}"; then + report_error "${crate} ${required_file} does not resolve to the repository root ${required_file}: ${repo_path}" + status=1 + fi + done + + return "${status}" +} + +# Route CLI args +while [ "$#" -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +require_command cargo +require_command jq + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +cd "${REPO_ROOT}" + +FAILED=0 + +echo "Checking repository root..." +for required_file in "${REQUIRED_FILES[@]}"; do + if ! check_file_exists "${REPO_ROOT}/${required_file}"; then + report_error "Repository root ${required_file} does not resolve to a non-empty file: ${REPO_ROOT}/${required_file}" + FAILED=1 + fi +done + +# Check for non-zero exit code, otherwise store in `PUBLISHABLE_CRATES` +# Must be checked here rather than piped directly, otherwise the error code is ignored +if ! PUBLISHABLE_CRATES="$(publishable_crates)"; then + report_error "Could not list the publishable crates" + exit 1 +fi + +# Verify at least one crate was listed +if [ -z "${PUBLISHABLE_CRATES}" ]; then + report_error "No publishable crates found in the workspace, expected at least one" + exit 1 +fi + +while IFS=$'\t' read -r crate manifest_path; do + check_crate "${crate}" "$(dirname "${manifest_path}")" || FAILED=1 +done <<<"${PUBLISHABLE_CRATES}" + +if [ "${FAILED}" -ne 0 ]; then + echo "Every publishable crate must package a top-level LICENSE and NOTICE resolving to the repository root copies." >&2 + exit 1 +fi + +echo "All publishable crates package LICENSE and NOTICE." From 0f37499bec2e8be9724887808de0a8aa194f0450 Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 2 Sep 2026 11:27:31 +0100 Subject: [PATCH 19/74] refactor(scan): extract reusable scan functions (#3093) * refactor(scan): extract scan planning utilities * comments * comments * remove allocation --- crates/iceberg/public-api.txt | 7 +- crates/iceberg/src/scan/mod.rs | 173 ++++++++++---------- crates/iceberg/src/spec/name_mapping/mod.rs | 19 ++- crates/iceberg/src/spec/table_metadata.rs | 71 +++++++- crates/iceberg/src/spec/table_properties.rs | 58 +++++++ 5 files changed, 236 insertions(+), 92 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index d61912c906..4aff5d9da2 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -2270,6 +2270,9 @@ pub fn iceberg::spec::NameMapping::eq(&self, other: &iceberg::spec::NameMapping) impl core::fmt::Debug for iceberg::spec::NameMapping pub fn iceberg::spec::NameMapping::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::spec::NameMapping +impl core::str::traits::FromStr for iceberg::spec::NameMapping +pub type iceberg::spec::NameMapping::Err = iceberg::Error +pub fn iceberg::spec::NameMapping::from_str(value: &str) -> iceberg::Result impl serde_core::ser::Serialize for iceberg::spec::NameMapping pub fn iceberg::spec::NameMapping::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NameMapping @@ -2730,6 +2733,7 @@ pub fn iceberg::spec::TableMetadata::sort_orders_iter(&self) -> impl core::iter: pub fn iceberg::spec::TableMetadata::statistics_for_snapshot(&self, snapshot_id: i64) -> core::option::Option<&iceberg::spec::StatisticsFile> pub fn iceberg::spec::TableMetadata::statistics_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator pub fn iceberg::spec::TableMetadata::table_properties(&self) -> iceberg::spec::TableProperties<'_> +pub fn iceberg::spec::TableMetadata::unified_partition_type(&self, schema: &iceberg::spec::Schema) -> iceberg::Result pub fn iceberg::spec::TableMetadata::uuid(&self) -> uuid::Uuid pub async fn iceberg::spec::TableMetadata::write_to(&self, file_io: &iceberg::io::FileIO, metadata_location: &iceberg::MetadataLocation) -> iceberg::Result<()> impl core::clone::Clone for iceberg::spec::TableMetadata @@ -2814,6 +2818,7 @@ pub const iceberg::spec::TableProperties<'_>::PROPERTY_DATAFUSION_WRITE_FANOUT_E pub const iceberg::spec::TableProperties<'_>::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_FILE_FORMAT: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_NAME_MAPPING: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_PARTITION_SPEC: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_SORT_ORDER: &'static str pub const iceberg::spec::TableProperties<'_>::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str @@ -2875,6 +2880,7 @@ pub fn iceberg::spec::TableProperties<'properties>::commit_max_retry_wait_ms(&se pub fn iceberg::spec::TableProperties<'properties>::commit_min_retry_wait_ms(&self) -> iceberg::Result pub fn iceberg::spec::TableProperties<'properties>::commit_num_retries(&self) -> iceberg::Result pub fn iceberg::spec::TableProperties<'properties>::commit_total_retry_timeout_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::default_name_mapping(&self) -> iceberg::Result> pub fn iceberg::spec::TableProperties<'properties>::encryption_data_key_length(&self) -> iceberg::Result pub fn iceberg::spec::TableProperties<'properties>::encryption_key_id(&self) -> iceberg::Result> pub fn iceberg::spec::TableProperties<'properties>::gc_enabled(&self) -> iceberg::Result @@ -3086,7 +3092,6 @@ pub fn iceberg::spec::ViewVersionLog::serialize<__S>(&self, __serializer: __S) - impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ViewVersionLog pub fn iceberg::spec::ViewVersionLog::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub const iceberg::spec::DEFAULT_SCHEMA_ID: iceberg::spec::SchemaId -pub const iceberg::spec::DEFAULT_SCHEMA_NAME_MAPPING: &str pub const iceberg::spec::INITIAL_ROW_ID: u64 pub const iceberg::spec::LIST_FIELD_NAME: &str pub const iceberg::spec::MAIN_BRANCH: &str diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 7abbf2dddf..909edf40ba 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -40,9 +40,8 @@ use crate::io::FileIO; use crate::metadata_columns::{ RESERVED_FIELD_ID_PARTITION, get_metadata_field_id, is_metadata_column_name, }; -use crate::partitioning::compute_unified_partition_type; use crate::runtime::Runtime; -use crate::spec::{DEFAULT_SCHEMA_NAME_MAPPING, DataContentType, NameMapping, Schema, SnapshotRef}; +use crate::spec::{DataContentType, Schema, SchemaRef, SnapshotRef, StructType}; use crate::table::Table; use crate::util::available_parallelism; use crate::{Error, ErrorKind, Result}; @@ -61,6 +60,73 @@ fn resolve_field_id(schema: &Schema, column_name: &str, case_sensitive: bool) -> } } +fn collect_scan_field_ids( + schema: &Schema, + column_names: Option<&[String]>, + case_sensitive: bool, +) -> Result> { + let Some(column_names) = column_names else { + return Ok(schema.as_struct().fields().iter().map(|f| f.id).collect()); + }; + + column_names + .iter() + .map(|column_name| { + if is_metadata_column_name(column_name) { + return get_metadata_field_id(column_name); + } + + let field_id = resolve_field_id(schema, column_name, case_sensitive).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Column {column_name} not found in table. Schema: {schema}"), + ) + })?; + + schema + .as_struct() + .field_by_id(field_id) + .ok_or_else(|| { + Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Column {column_name} is not a direct child of schema but a nested field, which is not supported now. Schema: {schema}" + ), + ) + })?; + + Ok(field_id) + }) + .collect() +} + +fn bind_scan_predicate( + schema: &SchemaRef, + predicate: Option<&Predicate>, + case_sensitive: bool, +) -> Result>> { + predicate + .map(|predicate| predicate.bind(schema.clone(), case_sensitive)) + .transpose() + .map(|predicate| predicate.map(Arc::new)) +} + +fn projected_partition_type( + table: &Table, + schema: &Schema, + field_ids: &[i32], +) -> Result>> { + if !field_ids.contains(&RESERVED_FIELD_ID_PARTITION) { + return Ok(None); + } + + table + .metadata() + .unified_partition_type(schema) + .map(Arc::new) + .map(Some) +} + /// Builder to create table scan. pub struct TableScanBuilder<'a> { table: &'a Table, @@ -233,85 +299,17 @@ impl<'a> TableScanBuilder<'a> { }; let schema = snapshot.schema(self.table.metadata())?; - - let mut field_ids = vec![]; - let column_names = self.column_names.clone().unwrap_or_else(|| { - schema - .as_struct() - .fields() - .iter() - .map(|f| f.name.clone()) - .collect() - }); - - for column_name in column_names.iter() { - // Handle metadata columns (like "_file") - if is_metadata_column_name(column_name) { - field_ids.push(get_metadata_field_id(column_name)?); - continue; - } - - let field_id = - resolve_field_id(&schema, column_name, self.case_sensitive).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Column {column_name} not found in table. Schema: {schema}"), - ) - })?; - - schema - .as_struct() - .field_by_id(field_id) - .ok_or_else(|| { - Error::new( - ErrorKind::FeatureUnsupported, - format!( - "Column {column_name} is not a direct child of schema but a nested field, which is not supported now. Schema: {schema}" - ), - ) - })?; - - field_ids.push(field_id); - } - - let snapshot_bound_predicate = if let Some(ref predicates) = self.filter { - Some(predicates.bind(schema.clone(), self.case_sensitive)?) - } else { - None - }; - + let field_ids = + collect_scan_field_ids(&schema, self.column_names.as_deref(), self.case_sensitive)?; + let snapshot_bound_predicate = + bind_scan_predicate(&schema, self.filter.as_ref(), self.case_sensitive)?; let name_mapping = self .table .metadata() - .properties() - .get(DEFAULT_SCHEMA_NAME_MAPPING) - .map(|raw| { - serde_json::from_str::(raw).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Failed to parse table property {DEFAULT_SCHEMA_NAME_MAPPING} as a NameMapping" - ), - ) - .with_source(e) - }) - }) - .transpose()? + .table_properties() + .default_name_mapping()? .map(Arc::new); - - // Compute unified partition type if _partition is projected - let unified_partition_type = if field_ids.contains(&RESERVED_FIELD_ID_PARTITION) { - let partition_type = compute_unified_partition_type( - self.table - .metadata() - .partition_specs_iter() - .map(|s| s.as_ref()), - &schema, - )?; - Some(Arc::new(partition_type)) - } else { - None - }; + let unified_partition_type = projected_partition_type(self.table, &schema, &field_ids)?; let plan_context = PlanContext { snapshot, @@ -319,7 +317,7 @@ impl<'a> TableScanBuilder<'a> { snapshot_schema: schema, case_sensitive: self.case_sensitive, predicate: self.filter.map(Arc::new), - snapshot_bound_predicate: snapshot_bound_predicate.map(Arc::new), + snapshot_bound_predicate, object_cache: self.table.object_cache(), field_ids: Arc::new(field_ids), name_mapping, @@ -664,11 +662,10 @@ pub mod tests { }; use crate::scan::FileScanTask; use crate::spec::{ - DEFAULT_SCHEMA_NAME_MAPPING, DataContentType, DataFileBuilder, DataFileFormat, Datum, - FormatVersion, Literal, MAIN_BRANCH, ManifestEntry, ManifestListWriter, ManifestStatus, - ManifestWriterBuilder, NestedField, Operation, PartitionSpec, PrimitiveType, Schema, - Snapshot, Struct, StructType, Summary, TableMetadata, TableMetadataBuilder, Type, - UnboundPartitionSpec, + DataContentType, DataFileBuilder, DataFileFormat, Datum, FormatVersion, Literal, + MAIN_BRANCH, ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, + NestedField, Operation, PartitionSpec, PrimitiveType, Schema, Snapshot, Struct, StructType, + Summary, TableMetadata, TableMetadataBuilder, TableProperties, Type, UnboundPartitionSpec, }; use crate::table::Table; use crate::test_utils::test_runtime; @@ -1896,7 +1893,8 @@ pub mod tests { #[test] fn test_table_scan_with_name_mapping_property() { let mapping_json = r#"[{"field-id":1,"names":["id","record_id"]}]"#; - let table = table_with_property(DEFAULT_SCHEMA_NAME_MAPPING, mapping_json); + let table = + table_with_property(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING, mapping_json); let table_scan = table.scan().build().unwrap(); let mapping = table_scan @@ -1917,7 +1915,10 @@ pub mod tests { #[test] fn test_table_scan_with_malformed_name_mapping_property() { - let table = table_with_property(DEFAULT_SCHEMA_NAME_MAPPING, "{ not valid json"); + let table = table_with_property( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING, + "{ not valid json", + ); let err = table .scan() @@ -1934,7 +1935,7 @@ pub mod tests { let mapping_json = r#"[{"field-id":1,"names":["id","record_id"]}]"#; let mut metadata = fixture.table.metadata().clone(); metadata.properties.insert( - DEFAULT_SCHEMA_NAME_MAPPING.to_string(), + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), mapping_json.to_string(), ); let table = Table::builder() diff --git a/crates/iceberg/src/spec/name_mapping/mod.rs b/crates/iceberg/src/spec/name_mapping/mod.rs index db9e44c290..a87e829ad0 100644 --- a/crates/iceberg/src/spec/name_mapping/mod.rs +++ b/crates/iceberg/src/spec/name_mapping/mod.rs @@ -17,13 +17,13 @@ //! Iceberg name mapping. +use std::str::FromStr; use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; -/// Property name for name mapping. -pub const DEFAULT_SCHEMA_NAME_MAPPING: &str = "schema.name-mapping.default"; +use crate::{Error, ErrorKind, Result}; /// Iceberg fallback field name to ID mapping. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] @@ -44,6 +44,21 @@ impl NameMapping { } } +impl FromStr for NameMapping { + type Err = Error; + + /// Parses a [`NameMapping`] from its JSON representation. + fn from_str(value: &str) -> Result { + serde_json::from_str(value).map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + "Failed to parse value as a NameMapping", + ) + .with_source(error) + }) + } +} + /// Maps field names to IDs. #[serde_as] #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 92bb6ff4d8..2b9d1d140e 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -33,14 +33,15 @@ use uuid::Uuid; use super::snapshot::SnapshotReference; pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataBuilder}; use super::{ - DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, - SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, + DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, Schema, SchemaId, + SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, TableProperties, }; use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; use crate::compression::CompressionCodec; use crate::error::{Result, timestamp_ms_to_utc}; use crate::io::FileIO; +use crate::partitioning::compute_unified_partition_type; use crate::spec::EncryptedKey; use crate::{Error, ErrorKind}; @@ -277,6 +278,19 @@ impl TableMetadata { &self.default_partition_type } + /// Returns the unified partition type across every partition spec in the table, resolved + /// against `schema`. + /// + /// Unlike [`Self::default_partition_type`], the result contains all partition fields ever + /// used by the table, so partition values stay readable across partition spec evolution. + /// See [`compute_unified_partition_type`] for the exact merge rules. + pub fn unified_partition_type(&self, schema: &Schema) -> Result { + compute_unified_partition_type( + self.partition_specs_iter().map(|spec| spec.as_ref()), + schema, + ) + } + #[inline] /// Returns spec id of the "current" partition spec. pub fn default_partition_spec_id(&self) -> i32 { @@ -1637,7 +1651,7 @@ mod tests { BlobMetadata, EncryptedKey, INITIAL_ROW_ID, Literal, NestedField, NullOrder, Operation, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot, SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, StatisticsFile, - Summary, TableProperties, Transform, Type, UnboundPartitionField, + Summary, TableProperties, Transform, Type, UnboundPartitionField, UnboundPartitionSpec, }; use crate::{ErrorKind, TableCreation}; @@ -4433,4 +4447,55 @@ mod tests { "s3://other-bucket/custom-meta" ); } + + #[test] + fn test_unified_partition_type_spans_all_specs() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let metadata = TableMetadataBuilder::new( + schema.clone(), + UnboundPartitionSpec::builder() + .with_spec_id(0) + .add_partition_field(2, "y", Transform::Identity) + .unwrap() + .build(), + SortOrder::unsorted_order(), + "s3://bucket/table".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata + .into_builder(None) + .add_partition_spec( + UnboundPartitionSpec::builder() + .add_partition_field(3, "z", Transform::Identity) + .unwrap() + .build(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + // The default spec only knows about `y`, but `_partition` must expose both. + assert_eq!(metadata.default_partition_type().fields().len(), 1); + + let unified = metadata.unified_partition_type(&schema).unwrap(); + let names: Vec<&str> = unified + .fields() + .iter() + .map(|field| field.name.as_str()) + .collect(); + assert_eq!(names, vec!["y", "z"]); + } } diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index b927d36386..01a330e28d 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -22,6 +22,7 @@ use iceberg_property_macro::properties_view; use crate::compression::CompressionCodec; use crate::encryption::AesKeySize; use crate::error::{Error, ErrorKind, Result}; +use crate::spec::NameMapping; use crate::util::location::strip_trailing_slash; fn parse_location_property(path: &str) -> Result { @@ -336,6 +337,14 @@ pub struct TableProperties { getter )] write_object_storage_partitioned_paths: bool, + /// The table's default name mapping, used to assign field ids when reading data files + /// that carry no field id metadata. `None` if `schema.name-mapping.default` is not set. + #[property( + key = Self::PROPERTY_DEFAULT_NAME_MAPPING, + default = None, + getter + )] + default_name_mapping: Option, } } @@ -432,6 +441,10 @@ impl TableProperties<'_> { /// location. pub const PROPERTY_WRITE_METADATA_PATH: &'static str = "write.metadata.path"; + /// Property key for the table's default name mapping, stored as a JSON + /// [`NameMapping`] document. + pub const PROPERTY_DEFAULT_NAME_MAPPING: &'static str = "schema.name-mapping.default"; + /// Compression codec for metadata files (JSON) pub const PROPERTY_METADATA_COMPRESSION_CODEC: &'static str = "write.metadata.compression-codec"; @@ -1162,4 +1175,49 @@ mod tests { assert!(tp.write_object_storage_partitioned_paths().unwrap()); } } + + #[test] + fn test_table_properties_default_name_mapping() { + // Test unset. + let properties = HashMap::new(); + assert!( + TableProperties::new(&properties) + .default_name_mapping() + .unwrap() + .is_none() + ); + + let properties = HashMap::from([( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), + r#"[{"field-id":1,"names":["id","record_id"]}]"#.to_string(), + )]); + let mapping = TableProperties::new(&properties) + .default_name_mapping() + .unwrap() + .unwrap(); + assert_eq!(mapping.fields().len(), 1); + assert_eq!(mapping.fields()[0].field_id(), Some(1)); + assert_eq!(mapping.fields()[0].names(), &[ + "id".to_string(), + "record_id".to_string() + ]); + } + + #[test] + fn test_table_properties_malformed_name_mapping() { + let properties = HashMap::from([( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), + "{ not valid json".to_string(), + )]); + let error = TableProperties::new(&properties) + .default_name_mapping() + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + // The property key must survive as error context. + assert!( + format!("{error}").contains(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING), + "{error}" + ); + } } From a18a0b20f338b1c6512df3da29aa19136015c45b Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:38:35 +0800 Subject: [PATCH 20/74] feat(io): make FileIO serializable (#3090) --- Cargo.lock | 1 + crates/iceberg/public-api.txt | 2 + crates/iceberg/src/io/file_io.rs | 133 ++++++++++++++++++ crates/storage/opendal/Cargo.toml | 1 + crates/storage/opendal/src/lib.rs | 59 +++++++- crates/storage/opendal/src/resolving.rs | 42 +++++- .../storage/opendal/tests/file_io_fs_test.rs | 51 +++++++ .../storage/opendal/tests/file_io_gcs_test.rs | 25 ++++ .../storage/opendal/tests/file_io_hf_test.rs | 9 +- .../opendal/tests/file_io_memory_test.rs | 48 +++++++ .../storage/opendal/tests/file_io_s3_test.rs | 29 ++++ .../opendal/tests/resolving_storage_test.rs | 7 +- 12 files changed, 402 insertions(+), 5 deletions(-) create mode 100644 crates/storage/opendal/tests/file_io_fs_test.rs create mode 100644 crates/storage/opendal/tests/file_io_memory_test.rs diff --git a/Cargo.lock b/Cargo.lock index 326457640c..2ebec585a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4064,6 +4064,7 @@ dependencies = [ "reqsign-core", "reqwest 0.12.28", "serde", + "tempfile", "tokio", "typetag", "url", diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 4aff5d9da2..3a62f733de 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -730,11 +730,13 @@ pub fn iceberg::io::FileIO::config(&self) -> &iceberg::io::StorageConfig pub async fn iceberg::io::FileIO::delete(&self, path: impl core::convert::AsRef) -> iceberg::Result<()> pub async fn iceberg::io::FileIO::delete_prefix(&self, path: impl core::convert::AsRef) -> iceberg::Result<()> pub async fn iceberg::io::FileIO::delete_stream(&self, paths: impl futures_core::stream::Stream + core::marker::Send + 'static) -> iceberg::Result<()> +pub fn iceberg::io::FileIO::deserialize_all(bytes: &[u8]) -> iceberg::Result pub async fn iceberg::io::FileIO::exists(&self, path: impl core::convert::AsRef) -> iceberg::Result pub fn iceberg::io::FileIO::new_input(&self, path: impl core::convert::AsRef) -> iceberg::Result pub fn iceberg::io::FileIO::new_output(&self, path: impl core::convert::AsRef) -> iceberg::Result pub fn iceberg::io::FileIO::new_with_fs() -> Self pub fn iceberg::io::FileIO::new_with_memory() -> Self +pub fn iceberg::io::FileIO::serialize_all(&self) -> iceberg::Result> impl core::clone::Clone for iceberg::io::FileIO pub fn iceberg::io::FileIO::clone(&self) -> iceberg::io::FileIO impl core::fmt::Debug for iceberg::io::FileIO diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index cd0a4434c4..87ac84784f 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -69,6 +69,26 @@ pub struct FileIO { storage: Arc>>, } +mod _serde { + use std::sync::Arc; + + use serde::{Deserialize, Serialize}; + + use super::{StorageConfig, StorageFactory}; + + #[derive(Serialize)] + pub(super) struct SerializableFileIO<'a> { + pub(super) config: &'a StorageConfig, + pub(super) factory: &'a Arc, + } + + #[derive(Deserialize)] + pub(super) struct DeserializedFileIO { + pub(super) config: StorageConfig, + pub(super) factory: Arc, + } +} + impl FileIO { /// Create a new FileIO backed by in-memory storage. /// @@ -92,6 +112,42 @@ impl FileIO { } } + /// Serializes all portable state of this `FileIO` into a byte vector. + /// + /// This includes the storage configuration and factory, but not the cached storage instance. + /// The storage cache is rebuilt lazily on first use after calling [`FileIO::deserialize_all`]. + /// + /// The serialized representation is not a stable format and may change between crate versions. + /// Applications should not rely on it for long-term storage or exchange it between incompatible + /// versions of this crate. + /// + /// All storage configuration properties are included in the serialized representation. These + /// properties may contain credentials or other sensitive values, so the returned bytes must be + /// protected in transit and at rest by the application embedding this crate. + /// + /// Storage factories are serialized through [`typetag`](https://docs.rs/typetag). Third-party + /// factories must use `#[typetag::serde]` on their [`StorageFactory`] implementation. + pub fn serialize_all(&self) -> Result> { + Ok(serde_json::to_vec(&_serde::SerializableFileIO { + config: &self.config, + factory: &self.factory, + })?) + } + + /// Deserializes a `FileIO` previously produced by [`FileIO::serialize_all`]. + /// + /// The receiving binary must use a compatible crate version and link the concrete factory + /// implementation so it is registered with `typetag`. Backend-specific requirements are + /// documented by each storage factory implementation. + pub fn deserialize_all(bytes: &[u8]) -> Result { + let _serde::DeserializedFileIO { config, factory } = serde_json::from_slice(bytes)?; + Ok(Self { + config, + factory, + storage: Arc::new(OnceLock::new()), + }) + } + /// Get the storage configuration. pub fn config(&self) -> &StorageConfig { &self.config @@ -544,4 +600,81 @@ mod tests { assert_eq!(file_io.config().get("key1"), Some(&"value1".to_string())); assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string())); } + + #[tokio::test] + async fn test_memory_file_io_serialization_roundtrip() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("test-property", "test-value") + .with_prop("s3.session-token", "test-token") + .build(); + + file_io + .new_output("memory://test/file.txt") + .unwrap() + .write("test".into()) + .await + .unwrap(); + assert!(file_io.storage.get().is_some()); + + let serialized = file_io.serialize_all().unwrap(); + let deserialized = FileIO::deserialize_all(&serialized).unwrap(); + assert!(deserialized.storage.get().is_none()); + assert_eq!( + deserialized.config().get("test-property"), + Some(&"test-value".to_string()) + ); + assert_eq!( + deserialized.config().get("s3.session-token"), + Some(&"test-token".to_string()) + ); + + deserialized + .new_output("memory://test/roundtrip.txt") + .unwrap() + .write("roundtrip".into()) + .await + .unwrap(); + assert_eq!( + deserialized + .new_input("memory://test/roundtrip.txt") + .unwrap() + .read() + .await + .unwrap(), + Bytes::from("roundtrip") + ); + assert!(deserialized.storage.get().is_some()); + } + + #[tokio::test] + async fn test_local_fs_file_io_serialization_roundtrip() { + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("roundtrip.txt"); + let path = path.to_str().unwrap(); + let file_io = FileIOBuilder::new(Arc::new(LocalFsStorageFactory)) + .with_prop("test-property", "test-value") + .build(); + + file_io + .new_output(path) + .unwrap() + .write("roundtrip".into()) + .await + .unwrap(); + assert!(file_io.storage.get().is_some()); + + let serialized = file_io.serialize_all().unwrap(); + let deserialized = FileIO::deserialize_all(&serialized).unwrap(); + assert!(deserialized.storage.get().is_none()); + assert_eq!( + deserialized.config().get("test-property"), + Some(&"test-value".to_string()) + ); + assert!(deserialized.exists(path).await.unwrap()); + assert_eq!( + deserialized.new_input(path).unwrap().read().await.unwrap(), + Bytes::from("roundtrip") + ); + assert!(deserialized.storage.get().is_some()); + } } diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index e43e7845b3..c6ea9f9b67 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -65,6 +65,7 @@ url = { workspace = true } async-trait = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } reqwest = { workspace = true } +tempfile = { workspace = true } tokio = { workspace = true, features = ["macros"] } [lints] diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index d02507e7b5..2ed5a60cb1 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -104,6 +104,15 @@ pub use resolving::{OpenDalResolvingStorage, OpenDalResolvingStorageFactory}; /// /// Maps scheme to the corresponding OpenDalStorage storage variant. /// Use this factory with `FileIOBuilder::new(factory)` to create FileIO instances. +/// +/// # Serialization +/// +/// The receiving binary must enable the feature corresponding to the serialized backend variant. +/// For example, deserializing `OpenDalStorageFactory::S3` requires the `opendal-s3` feature. +/// +/// Serialization fails when the `OpenDalStorageFactory::S3` variant contains a custom AWS +/// credential loader because the loader holds process-local state that cannot be reconstructed in +/// another process. Construct the factory without a custom loader before serializing it. #[derive(Clone, Debug, Serialize, Deserialize)] pub enum OpenDalStorageFactory { /// Memory storage factory. @@ -116,7 +125,11 @@ pub enum OpenDalStorageFactory { #[cfg(feature = "opendal-s3")] S3 { /// Custom AWS credential loader. - #[serde(skip)] + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_custom_credential_loader" + )] customized_credential_load: Option, }, /// GCS storage factory. @@ -133,6 +146,19 @@ pub enum OpenDalStorageFactory { Hf, } +#[cfg(feature = "opendal-s3")] +pub(crate) fn serialize_custom_credential_loader( + _loader: &Option, + _serializer: S, +) -> std::result::Result +where + S: serde::Serializer, +{ + Err(serde::ser::Error::custom( + "custom AWS credential loaders cannot be serialized", + )) +} + #[typetag::serde(name = "OpenDalStorageFactory")] impl StorageFactory for OpenDalStorageFactory { #[allow(unused_variables)] @@ -631,6 +657,37 @@ impl FileWrite for OpenDalWriter { mod tests { use super::*; + #[cfg(feature = "opendal-s3")] + #[derive(Debug)] + struct EmptyCredentialLoader; + + #[cfg(feature = "opendal-s3")] + impl ProvideCredential for EmptyCredentialLoader { + type Credential = AwsCredential; + + async fn provide_credential( + &self, + _ctx: &reqsign_core::Context, + ) -> reqsign_core::Result> { + Ok(None) + } + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_s3_factory_custom_credential_loader_serialization_fails() { + let file_io = iceberg::io::FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: Some(CustomAwsCredentialLoader::new(EmptyCredentialLoader)), + })) + .build(); + + let err = file_io.serialize_all().unwrap_err(); + assert!( + err.to_string() + .contains("custom AWS credential loaders cannot be serialized") + ); + } + #[cfg(feature = "opendal-memory")] #[test] fn test_default_memory_operator() { diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 86993220a8..3b99b08fc0 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -140,6 +140,12 @@ fn build_storage_for_scheme( /// delegates operations to the appropriate [`OpenDalStorage`] variant based on /// the path scheme. /// +/// # Serialization +/// +/// Serialization fails when a custom S3 credential loader is configured because the loader holds +/// process-local state that cannot be reconstructed in another process. Construct the factory +/// without a custom loader before serializing it. +/// /// # Example /// /// ```rust,ignore @@ -156,7 +162,11 @@ fn build_storage_for_scheme( pub struct OpenDalResolvingStorageFactory { /// Custom AWS credential loader for S3 storage. #[cfg(feature = "opendal-s3")] - #[serde(skip)] + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "crate::serialize_custom_credential_loader" + )] customized_credential_load: Option, } @@ -326,6 +336,36 @@ impl Storage for OpenDalResolvingStorage { mod tests { use super::*; + #[cfg(feature = "opendal-s3")] + #[derive(Debug)] + struct EmptyCredentialLoader; + + #[cfg(feature = "opendal-s3")] + impl crate::s3::ProvideCredential for EmptyCredentialLoader { + type Credential = crate::s3::AwsCredential; + + async fn provide_credential( + &self, + _ctx: &reqsign_core::Context, + ) -> reqsign_core::Result> { + Ok(None) + } + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_custom_credential_loader_serialization_fails() { + let factory = OpenDalResolvingStorageFactory::new() + .with_s3_credential_loader(CustomAwsCredentialLoader::new(EmptyCredentialLoader)); + let file_io = iceberg::io::FileIOBuilder::new(Arc::new(factory)).build(); + + let err = file_io.serialize_all().unwrap_err(); + assert!( + err.to_string() + .contains("custom AWS credential loaders cannot be serialized") + ); + } + /// Builds a resolving storage with empty props, suitable for `resolve()` /// calls that don't actually hit any backend. fn empty_resolving_storage() -> OpenDalResolvingStorage { diff --git a/crates/storage/opendal/tests/file_io_fs_test.rs b/crates/storage/opendal/tests/file_io_fs_test.rs new file mode 100644 index 0000000000..f9bb82db7e --- /dev/null +++ b/crates/storage/opendal/tests/file_io_fs_test.rs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for FileIO OpenDAL filesystem storage. + +#[cfg(feature = "opendal-fs")] +mod tests { + use std::sync::Arc; + + use bytes::Bytes; + use iceberg::io::{FileIO, FileIOBuilder}; + use iceberg_storage_opendal::OpenDalStorageFactory; + use tempfile::TempDir; + + #[tokio::test] + async fn test_file_io_fs_serialization_roundtrip() { + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("serialization-roundtrip"); + let path = format!("file:/{}", path.display()); + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Fs)).build(); + let serialized = file_io.serialize_all().unwrap(); + let file_io = FileIO::deserialize_all(&serialized).unwrap(); + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"roundtrip")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"roundtrip") + ); + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } +} diff --git a/crates/storage/opendal/tests/file_io_gcs_test.rs b/crates/storage/opendal/tests/file_io_gcs_test.rs index b43cc3a535..c6ba770b17 100644 --- a/crates/storage/opendal/tests/file_io_gcs_test.rs +++ b/crates/storage/opendal/tests/file_io_gcs_test.rs @@ -47,6 +47,31 @@ mod tests { .build() } + fn roundtrip_file_io(file_io: &FileIO) -> FileIO { + let serialized = file_io.serialize_all().unwrap(); + FileIO::deserialize_all(&serialized).unwrap() + } + + #[tokio::test] + async fn test_file_io_gcs_serialization_roundtrip() { + let file_io = roundtrip_file_io(&get_file_io_gcs().await); + let path = format!("{}/serialization-roundtrip", get_gs_path()); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"roundtrip")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"roundtrip") + ); + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } + // Create a bucket against the emulated GCS storage server. async fn create_bucket(name: &str, server_endpoint: &str) -> anyhow::Result<()> { let mut bucket_data = HashMap::new(); diff --git a/crates/storage/opendal/tests/file_io_hf_test.rs b/crates/storage/opendal/tests/file_io_hf_test.rs index 44fe3420d4..f8d17e2ad1 100644 --- a/crates/storage/opendal/tests/file_io_hf_test.rs +++ b/crates/storage/opendal/tests/file_io_hf_test.rs @@ -68,13 +68,18 @@ mod tests { .build() } + fn roundtrip_file_io(file_io: &FileIO) -> FileIO { + let serialized = file_io.serialize_all().unwrap(); + FileIO::deserialize_all(&serialized).unwrap() + } + // --- bucket tests --- #[tokio::test] async fn test_hf_bucket_write_read_delete() { let token = require_env!(ENV_HF_TOKEN); let bucket = require_env!(ENV_HF_BUCKET); - let file_io = get_file_io(&token); + let file_io = roundtrip_file_io(&get_file_io(&token)); let path = format!( "hf://buckets/{}/{}", bucket, @@ -324,7 +329,7 @@ mod tests { #[tokio::test] async fn test_hf_resolving_storage() { let token = require_env!(ENV_HF_TOKEN); - let file_io = get_resolving_file_io(&token); + let file_io = roundtrip_file_io(&get_resolving_file_io(&token)); let bucket = require_env!(ENV_HF_BUCKET); let path = format!( diff --git a/crates/storage/opendal/tests/file_io_memory_test.rs b/crates/storage/opendal/tests/file_io_memory_test.rs new file mode 100644 index 0000000000..9c3c335a3f --- /dev/null +++ b/crates/storage/opendal/tests/file_io_memory_test.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for FileIO OpenDAL memory storage. + +#[cfg(feature = "opendal-memory")] +mod tests { + use std::sync::Arc; + + use bytes::Bytes; + use iceberg::io::{FileIO, FileIOBuilder}; + use iceberg_storage_opendal::OpenDalStorageFactory; + + #[tokio::test] + async fn test_file_io_memory_serialization_roundtrip() { + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Memory)).build(); + let serialized = file_io.serialize_all().unwrap(); + let file_io = FileIO::deserialize_all(&serialized).unwrap(); + let path = "memory://serialization-roundtrip"; + + file_io + .new_output(path) + .unwrap() + .write(Bytes::from_static(b"roundtrip")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(path).unwrap().read().await.unwrap(), + Bytes::from_static(b"roundtrip") + ); + file_io.delete(path).await.unwrap(); + assert!(!file_io.exists(path).await.unwrap()); + } +} diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index d5858e18f4..6b57f86607 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -23,6 +23,7 @@ mod tests { use std::sync::Arc; + use bytes::Bytes; use futures::StreamExt; use iceberg::io::{ FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, @@ -52,6 +53,34 @@ mod tests { .build() } + fn roundtrip_file_io(file_io: &FileIO) -> FileIO { + let serialized = file_io.serialize_all().unwrap(); + FileIO::deserialize_all(&serialized).unwrap() + } + + #[tokio::test] + async fn test_file_io_s3_serialization_roundtrip() { + let file_io = roundtrip_file_io(&get_file_io().await); + let path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_file_io_s3_serialization_roundtrip") + ); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"roundtrip")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"roundtrip") + ); + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } + #[tokio::test] async fn test_file_io_s3_exists() { let file_io = get_file_io().await; diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index 1853a796dd..ba94b8332d 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -51,6 +51,11 @@ mod tests { .build() } + fn roundtrip_file_io(file_io: &iceberg::io::FileIO) -> iceberg::io::FileIO { + let serialized = file_io.serialize_all().unwrap(); + iceberg::io::FileIO::deserialize_all(&serialized).unwrap() + } + fn temp_fs_path(name: &str) -> String { let dir = std::env::temp_dir().join("iceberg_resolving_tests"); std::fs::create_dir_all(&dir).unwrap(); @@ -62,7 +67,7 @@ mod tests { #[tokio::test] async fn test_mixed_scheme_write_and_read() { - let file_io = get_resolving_file_io(); + let file_io = roundtrip_file_io(&get_resolving_file_io()); let s3_path = format!( "s3://bucket1/{}", From 12575c7bdcff652ff23e8f97a4e4c6263c1caa0e Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:15:14 +0800 Subject: [PATCH 21/74] refactor(scan): validate FileScanTask construction (#3131) * refactor(scan): Validate FileScanTask construction Make FileScanTask fields private, expose read-only getters, and validate builder output so non-empty partition data always has an associated partition spec. Generated-by: Codex (GPT-5) * refactor(scan): Simplify FileScanTask validation Use typed-builder output conversion for validation instead of coupling the custom build method to generated typestate arguments. Generated-by: Codex (GPT-5) * fix(scan): Validate FileScanTask partition data Validate partition arity and transformed value types while preserving support for historical partition specs whose source columns were dropped. Port the partition coverage from PR #3091 to builder regression tests. Generated-by: Codex (GPT-5) * fix(scan): Refine partition validation Enforce symmetric partition/spec presence and tuple arity while avoiding per-value transform validation during task construction. Update the spec-id scan fixture to provide its partition tuple. Generated-by: Codex (GPT-5) * fix(scan): Validate partition specs against schema Preserve historical specs with dropped source columns while validating resolvable partition specs against the task schema. Add regression coverage for an incompatible evolved schema. Generated-by: Codex (GPT-5) * fix(scan): Always validate partition specs Remove the dropped-source shortcut and rely directly on PartitionSpec::partition_type for schema validation. Update dropped-source coverage to assert the resulting planning error. Generated-by: Codex (GPT-5) --- crates/iceberg/public-api.txt | 34 +- .../src/arrow/caching_delete_file_loader.rs | 20 +- .../iceberg/src/arrow/delete_file_loader.rs | 2 +- crates/iceberg/src/arrow/delete_filter.rs | 23 +- crates/iceberg/src/arrow/reader/pipeline.rs | 264 ++++++++------ .../src/arrow/reader/positional_deletes.rs | 9 +- crates/iceberg/src/arrow/reader/projection.rs | 36 +- crates/iceberg/src/arrow/reader/row_filter.rs | 82 ++--- crates/iceberg/src/scan/context.rs | 4 +- crates/iceberg/src/scan/mod.rs | 87 ++--- crates/iceberg/src/scan/task.rs | 327 ++++++++++++++++-- .../src/writer/file_writer/parquet_writer.rs | 3 +- .../tests/read_positional_deletes.rs | 2 +- 13 files changed, 601 insertions(+), 292 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 3a62f733de..95df9e9443 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1274,34 +1274,32 @@ pub const iceberg::puffin::CREATED_BY_PROPERTY: &str pub const iceberg::puffin::DELETION_VECTOR_V1: &str pub mod iceberg::scan pub struct iceberg::scan::FileScanTask -pub iceberg::scan::FileScanTask::case_sensitive: bool -pub iceberg::scan::FileScanTask::data_file_format: iceberg::spec::DataFileFormat -pub iceberg::scan::FileScanTask::data_file_path: alloc::string::String -pub iceberg::scan::FileScanTask::data_sequence_number: core::option::Option -pub iceberg::scan::FileScanTask::deletes: alloc::vec::Vec -pub iceberg::scan::FileScanTask::file_size_in_bytes: u64 -pub iceberg::scan::FileScanTask::first_row_id: core::option::Option -pub iceberg::scan::FileScanTask::key_metadata: core::option::Option> -pub iceberg::scan::FileScanTask::length: u64 -pub iceberg::scan::FileScanTask::name_mapping: core::option::Option> -pub iceberg::scan::FileScanTask::partition: core::option::Option -pub iceberg::scan::FileScanTask::partition_spec: core::option::Option> -pub iceberg::scan::FileScanTask::predicate: core::option::Option -pub iceberg::scan::FileScanTask::project_field_ids: alloc::vec::Vec -pub iceberg::scan::FileScanTask::record_count: core::option::Option -pub iceberg::scan::FileScanTask::schema: iceberg::spec::SchemaRef -pub iceberg::scan::FileScanTask::start: u64 -pub iceberg::scan::FileScanTask::unified_partition_type: core::option::Option> impl iceberg::scan::FileScanTask +pub fn iceberg::scan::FileScanTask::case_sensitive(&self) -> bool +pub fn iceberg::scan::FileScanTask::data_file_format(&self) -> iceberg::spec::DataFileFormat pub fn iceberg::scan::FileScanTask::data_file_path(&self) -> &str +pub fn iceberg::scan::FileScanTask::data_sequence_number(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTask::deletes(&self) -> &[iceberg::scan::FileScanTaskDeleteFile] +pub fn iceberg::scan::FileScanTask::file_size_in_bytes(&self) -> u64 +pub fn iceberg::scan::FileScanTask::first_row_id(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTask::key_metadata(&self) -> core::option::Option<&[u8]> +pub fn iceberg::scan::FileScanTask::length(&self) -> u64 +pub fn iceberg::scan::FileScanTask::name_mapping(&self) -> core::option::Option<&alloc::sync::Arc> +pub fn iceberg::scan::FileScanTask::partition(&self) -> core::option::Option<&iceberg::spec::Struct> +pub fn iceberg::scan::FileScanTask::partition_spec(&self) -> core::option::Option<&alloc::sync::Arc> pub fn iceberg::scan::FileScanTask::predicate(&self) -> core::option::Option<&iceberg::expr::BoundPredicate> pub fn iceberg::scan::FileScanTask::project_field_ids(&self) -> &[i32] +pub fn iceberg::scan::FileScanTask::record_count(&self) -> core::option::Option pub fn iceberg::scan::FileScanTask::schema(&self) -> &iceberg::spec::Schema pub fn iceberg::scan::FileScanTask::schema_ref(&self) -> iceberg::spec::SchemaRef +pub fn iceberg::scan::FileScanTask::start(&self) -> u64 +pub fn iceberg::scan::FileScanTask::unified_partition_type(&self) -> core::option::Option<&alloc::sync::Arc> impl core::clone::Clone for iceberg::scan::FileScanTask pub fn iceberg::scan::FileScanTask::clone(&self) -> iceberg::scan::FileScanTask impl core::cmp::PartialEq for iceberg::scan::FileScanTask pub fn iceberg::scan::FileScanTask::eq(&self, other: &iceberg::scan::FileScanTask) -> bool +impl core::convert::From for iceberg::Result +pub fn iceberg::Result::from(task: iceberg::scan::FileScanTask) -> Self impl core::fmt::Debug for iceberg::scan::FileScanTask pub fn iceberg::scan::FileScanTask::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::scan::FileScanTask diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 84b381a00f..896248ff6d 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -949,7 +949,10 @@ mod tests { let file_scan_tasks = setup(table_location); let delete_filter = delete_file_loader - .load_deletes(&file_scan_tasks[0].deletes, file_scan_tasks[0].schema_ref()) + .load_deletes( + file_scan_tasks[0].deletes(), + file_scan_tasks[0].schema_ref(), + ) .await .unwrap() .unwrap(); @@ -1268,13 +1271,14 @@ mod tests { .with_project_field_ids(vec![2, 3]) .with_deletes(vec![pos_del, eq_del]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); // Load the deletes - should handle both types without error let delete_file_loader = CachingDeleteFileLoader::new(file_io.clone(), 10, Runtime::current()); let delete_filter = delete_file_loader - .load_deletes(&file_scan_task.deletes, file_scan_task.schema_ref()) + .load_deletes(file_scan_task.deletes(), file_scan_task.schema_ref()) .await .unwrap() .unwrap(); @@ -1351,14 +1355,20 @@ mod tests { // Load deletes for the first time let delete_filter_1 = delete_file_loader - .load_deletes(&file_scan_tasks[0].deletes, file_scan_tasks[0].schema_ref()) + .load_deletes( + file_scan_tasks[0].deletes(), + file_scan_tasks[0].schema_ref(), + ) .await .unwrap() .unwrap(); // Load deletes for the second time (same task/files) let delete_filter_2 = delete_file_loader - .load_deletes(&file_scan_tasks[0].deletes, file_scan_tasks[0].schema_ref()) + .load_deletes( + file_scan_tasks[0].deletes(), + file_scan_tasks[0].schema_ref(), + ) .await .unwrap() .unwrap(); diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index 6a232cea0d..13fa9740ae 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -162,7 +162,7 @@ mod tests { let result = delete_file_loader .read_delete_file( - &file_scan_tasks[0].deletes[0], + &file_scan_tasks[0].deletes()[0], file_scan_tasks[0].schema_ref(), ) .await diff --git a/crates/iceberg/src/arrow/delete_filter.rs b/crates/iceberg/src/arrow/delete_filter.rs index bcfcbd233d..fb67d021e0 100644 --- a/crates/iceberg/src/arrow/delete_filter.rs +++ b/crates/iceberg/src/arrow/delete_filter.rs @@ -197,7 +197,7 @@ impl DeleteFilter { // * Bind the predicate to the task's schema to get a `BoundPredicate` let mut combined_predicate = AlwaysTrue; - for delete in &file_scan_task.deletes { + for delete in file_scan_task.deletes() { if !is_equality_delete(delete) { continue; } @@ -223,7 +223,7 @@ impl DeleteFilter { } let bound_predicate = combined_predicate - .bind(file_scan_task.schema.clone(), file_scan_task.case_sensitive)?; + .bind(file_scan_task.schema_ref(), file_scan_task.case_sensitive())?; Ok(Some(bound_predicate)) } @@ -345,7 +345,10 @@ pub(crate) mod tests { let file_scan_tasks = setup(table_location); let delete_filter = delete_file_loader - .load_deletes(&file_scan_tasks[0].deletes, file_scan_tasks[0].schema_ref()) + .load_deletes( + file_scan_tasks[0].deletes(), + file_scan_tasks[0].schema_ref(), + ) .await .unwrap() .unwrap(); @@ -356,7 +359,10 @@ pub(crate) mod tests { assert_eq!(result.lock().unwrap().len(), 12); // pos dels from pos del file 1 and 2 let delete_filter = delete_file_loader - .load_deletes(&file_scan_tasks[1].deletes, file_scan_tasks[1].schema_ref()) + .load_deletes( + file_scan_tasks[1].deletes(), + file_scan_tasks[1].schema_ref(), + ) .await .unwrap() .unwrap(); @@ -483,7 +489,8 @@ pub(crate) mod tests { .with_project_field_ids(vec![]) .with_deletes(vec![pos_del_1, pos_del_2.clone()]) .with_case_sensitive(false) - .build(), + .build() + .unwrap(), FileScanTask::builder() .with_file_size_in_bytes(0) .with_start(0) @@ -494,7 +501,8 @@ pub(crate) mod tests { .with_project_field_ids(vec![]) .with_deletes(vec![pos_del_3]) .with_case_sensitive(false) - .build(), + .build() + .unwrap(), ]; file_scan_tasks @@ -547,7 +555,8 @@ pub(crate) mod tests { .build(), ]) .with_case_sensitive(true) - .build(); + .build() + .unwrap(); let filter = DeleteFilter::new(Runtime::current()); diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 27728242e1..894e51d76d 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -129,23 +129,23 @@ struct FileScanTaskReader { impl FileScanTaskReader { async fn process(self, task: FileScanTask) -> Result { - let should_load_page_index = - (self.row_selection_enabled && task.predicate.is_some()) || !task.deletes.is_empty(); + let should_load_page_index = (self.row_selection_enabled && task.predicate().is_some()) + || !task.deletes().is_empty(); let mut parquet_read_options = self.parquet_read_options; parquet_read_options.preload_page_index = should_load_page_index; let delete_filter_rx = self .delete_file_loader - .load_deletes(&task.deletes, Arc::clone(&task.schema)); + .load_deletes(task.deletes(), task.schema_ref()); // Open the Parquet file once, loading its metadata let (parquet_file_reader, arrow_metadata) = ArrowReader::open_parquet_file( - &task.data_file_path, + task.data_file_path(), &self.file_io, - task.file_size_in_bytes, + task.file_size_in_bytes(), parquet_read_options, self.scan_metrics.bytes_read_counter(), - task.key_metadata.as_deref(), + task.key_metadata(), ) .await?; @@ -163,7 +163,7 @@ impl FileScanTaskReader { // AND no name mapping is available. With a name mapping, field IDs are assigned // to the Arrow schema below, and projection/predicate planning must use them // (see #2403). - let use_position_fallback = missing_field_ids && task.name_mapping.is_none(); + let use_position_fallback = missing_field_ids && task.name_mapping().is_none(); // Three-branch schema resolution strategy matching Java's ReadConf constructor // @@ -182,7 +182,7 @@ impl FileScanTaskReader { // - Branch 3: fallback → addFallbackIds(), then pruneColumnsFallback() let arrow_metadata = if missing_field_ids { // Parquet file lacks field IDs - must assign them before reading - let arrow_schema = if let Some(name_mapping) = &task.name_mapping { + let arrow_schema = if let Some(name_mapping) = task.name_mapping() { // Branch 2: Apply name mapping to assign correct Iceberg field IDs // Per spec rule #2: "Use schema.name-mapping.default metadata to map field id // to columns without field id" @@ -215,7 +215,7 @@ impl FileScanTaskReader { // Coerce INT96 timestamp columns to the resolution specified by the Iceberg schema. // This must happen before building the stream reader to avoid i64 overflow in arrow-rs. let arrow_metadata = if let Some(coerced_schema) = - coerce_int96_timestamps(arrow_metadata.schema(), &task.schema) + coerce_int96_timestamps(arrow_metadata.schema(), task.schema()) { let options = ArrowReaderOptions::new().with_schema(Arc::clone(&coerced_schema)); ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err( @@ -240,7 +240,7 @@ impl FileScanTaskReader { // positional fallback for `_row_id` (`first_row_id + pos`), so add it whenever // `_row_id` is synthesized. A null `first_row_id` nulls the whole `_row_id` // column, so nothing is synthesized and the column is not needed. - let need_row_number = project_pos || (project_row_id && task.first_row_id.is_some()); + let need_row_number = project_pos || (project_row_id && task.first_row_id().is_some()); let field_ids = task.project_field_ids(); let metadata_only_projection = !field_ids.is_empty() @@ -317,7 +317,7 @@ impl FileScanTaskReader { // column below, discarding any per-row values the file carries -- matching Java // (`ValueReaders.lastUpdated` nulls when the base row id is null). let coalesce_last_updated_seq_leaf = phys_last_updated_seq_leaf - .filter(|_| task.first_row_id.is_some() && task.data_sequence_number.is_some()); + .filter(|_| task.first_row_id().is_some() && task.data_sequence_number().is_some()); let phys_row_id_leaf = if project_row_id { find_leaf_by_field_id( @@ -344,11 +344,11 @@ impl FileScanTaskReader { // Read the physical column only when first_row_id is set. A null first_row_id // nulls the whole column below (matching Java `ValueReaders.rowIds`). - let coalesce_row_id_leaf = phys_row_id_leaf.filter(|_| task.first_row_id.is_some()); + let coalesce_row_id_leaf = phys_row_id_leaf.filter(|_| task.first_row_id().is_some()); // Filter out metadata fields for Parquet projection (they don't exist in files) let project_field_ids_without_metadata: Vec = task - .project_field_ids + .project_field_ids() .iter() .filter(|&&id| !is_metadata_field(id)) .copied() @@ -361,7 +361,7 @@ impl FileScanTaskReader { // - Otherwise: position-based fallback projection let mut projection_mask = ArrowReader::get_arrow_projection_mask( &project_field_ids_without_metadata, - &task.schema, + task.schema(), record_batch_stream_builder.parquet_schema(), record_batch_stream_builder.schema(), use_position_fallback, // Whether to use position-based (true) or field-ID-based (false) projection @@ -406,7 +406,7 @@ impl FileScanTaskReader { // Add the _file metadata column if it's in the projected fields if task.project_field_ids().contains(&RESERVED_FIELD_ID_FILE) { - let file_datum = Datum::string(task.data_file_path.clone()); + let file_datum = Datum::string(task.data_file_path().to_string()); record_batch_transformer_builder = record_batch_transformer_builder.with_constant(RESERVED_FIELD_ID_FILE, file_datum); } @@ -416,8 +416,7 @@ impl FileScanTaskReader { .contains(&RESERVED_FIELD_ID_SPEC_ID) { let partition_spec = task - .partition_spec - .as_ref() + .partition_spec() .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Partition spec is missing"))?; let spec_id_datum = Datum::int(partition_spec.spec_id()); @@ -430,52 +429,55 @@ impl FileScanTaskReader { // it this way (`ValueReaders.lastUpdated` returns nulls when the base row id is // null); the spec itself only says the column is assigned the manifest entry's // sequence number on read. - record_batch_transformer_builder = match (task.first_row_id, task.data_sequence_number) - { - (Some(_), Some(seq)) => { - let datum = Datum::long(seq); - if coalesce_last_updated_seq_leaf.is_some() { - // The file physically carries the column: read the per-row value, - // falling back to the data sequence number only where null. - record_batch_transformer_builder.with_coalesced_last_updated_seq_column( - RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, - datum, - ) - } else if last_updated_seq_present_by_name_only { - // Present by name but without the embedded field id (name mapping / - // positional fallback). The transformer keys the source column by - // field id, so we can't thread it; no real writer produces this, so - // reject loudly rather than silently overwrite with the constant. - // Arm-local by design: only this arm reads the physical column, so - // only here can a name-only column defeat us. The `(None, _)` arm - // nulls the column without reading it, so it needs no such guard. - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Reading a physically-stored _last_updated_sequence_number column \ + record_batch_transformer_builder = + match (task.first_row_id(), task.data_sequence_number()) { + (Some(_), Some(seq)) => { + let datum = Datum::long(seq); + if coalesce_last_updated_seq_leaf.is_some() { + // The file physically carries the column: read the per-row value, + // falling back to the data sequence number only where null. + record_batch_transformer_builder.with_coalesced_last_updated_seq_column( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + datum, + ) + } else if last_updated_seq_present_by_name_only { + // Present by name but without the embedded field id (name mapping / + // positional fallback). The transformer keys the source column by + // field id, so we can't thread it; no real writer produces this, so + // reject loudly rather than silently overwrite with the constant. + // Arm-local by design: only this arm reads the physical column, so + // only here can a name-only column defeat us. The `(None, _)` arm + // nulls the column without reading it, so it needs no such guard. + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Reading a physically-stored _last_updated_sequence_number column \ without an embedded field id is not supported", + )); + } else { + // Column absent: derive it from the data sequence number. + record_batch_transformer_builder.with_constant( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + datum, + ) + } + } + // Null first_row_id (v1/v2, or a pre-upgrade v3 snapshot): the column is null. + (None, _) => record_batch_transformer_builder.with_null_metadata_column( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + )?, + // first_row_id present but no data sequence number: after manifest + // inheritance a committed entry always has one, so this is a malformed + // manifest rather than a legitimate null. + (Some(_), None) => { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Data file {} has a first_row_id but no data sequence number", + task.data_file_path() + ), )); - } else { - // Column absent: derive it from the data sequence number. - record_batch_transformer_builder - .with_constant(RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, datum) } - } - // Null first_row_id (v1/v2, or a pre-upgrade v3 snapshot): the column is null. - (None, _) => record_batch_transformer_builder - .with_null_metadata_column(RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER)?, - // first_row_id present but no data sequence number: after manifest - // inheritance a committed entry always has one, so this is a malformed - // manifest rather than a legitimate null. - (Some(_), None) => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Data file {} has a first_row_id but no data sequence number", - task.data_file_path - ), - )); - } - }; + }; } if project_row_id { @@ -484,7 +486,7 @@ impl FileScanTaskReader { // leaf is never read and `_row_id` is nulled out downstream (matching Java // `ValueReaders.rowIds`), so a pre-v3 file with a user column named `_row_id` // reads back as null rather than erroring. - if task.first_row_id.is_some() && row_id_present_by_name_only { + if task.first_row_id().is_some() && row_id_present_by_name_only { return Err(Error::new( ErrorKind::FeatureUnsupported, "Reading a physically-stored _row_id column without an embedded field id \ @@ -500,10 +502,10 @@ impl FileScanTaskReader { } if let (Some(partition_spec), Some(partition_data)) = - (task.partition_spec.clone(), task.partition.clone()) + (task.partition_spec(), task.partition()) { - record_batch_transformer_builder = - record_batch_transformer_builder.with_partition(partition_spec, partition_data)?; + record_batch_transformer_builder = record_batch_transformer_builder + .with_partition(Arc::clone(partition_spec), partition_data.clone())?; } if project_pos { @@ -516,10 +518,10 @@ impl FileScanTaskReader { if task .project_field_ids() .contains(&RESERVED_FIELD_ID_PARTITION) - && let Some(unified_type) = &task.unified_partition_type + && let Some(unified_type) = task.unified_partition_type() { - let (spec, partition_data) = match (&task.partition_spec, &task.partition) { - (Some(spec), Some(data)) => (spec.clone(), data.clone()), + let (spec, partition_data) = match (task.partition_spec(), task.partition()) { + (Some(spec), Some(data)) => (Arc::clone(spec), data.clone()), // A missing spec/data is only acceptable when there are no partition // fields to fill (unpartitioned table). If the unified type has fields // but we lack a spec or data, the task is inconsistent and we cannot @@ -553,7 +555,7 @@ impl FileScanTaskReader { // we also have an optional predicate resulting from equality delete files. // If both are present, we logical-AND them together to form a single filter // predicate that we can pass to the `RecordBatchStreamBuilder`. - let final_predicate = match (&task.predicate, delete_predicate) { + let final_predicate = match (task.predicate(), delete_predicate) { (None, None) => None, (Some(predicate), None) => Some(predicate.clone()), (None, Some(ref predicate)) => Some(predicate.clone()), @@ -582,11 +584,11 @@ impl FileScanTaskReader { // Filter row groups based on byte range from task.start and task.length. // If both start and length are 0, read the entire file (backwards compatibility). - if task.start != 0 || task.length != 0 { + if task.start() != 0 || task.length() != 0 { let byte_range_filtered_row_groups = ArrowReader::filter_row_groups_by_byte_range( record_batch_stream_builder.metadata(), - task.start, - task.length, + task.start(), + task.length(), )?; selected_row_group_indices = Some(byte_range_filtered_row_groups); } @@ -612,7 +614,7 @@ impl FileScanTaskReader { &predicate, record_batch_stream_builder.metadata(), &field_id_map, - &task.schema, + task.schema(), )?; // Merge predicate-based filtering with byte range filtering (if present) @@ -636,7 +638,7 @@ impl FileScanTaskReader { record_batch_stream_builder.metadata(), &selected_row_group_indices, &field_id_map, - &task.schema, + task.schema(), )?; } } @@ -678,7 +680,7 @@ impl FileScanTaskReader { // to the requester. When `_row_id` is projected, synthesize it over the raw parquet // batches (using the reader-produced `_pos` position) before the transformer, which // then passes it through as a virtual field. - let first_row_id = task.first_row_id; + let first_row_id = task.first_row_id(); let record_batch_stream = record_batch_stream_builder.build()?.map(move |batch| { let mut batch = batch.map_err(|err| -> Error { err.into() })?; if project_row_id { @@ -793,7 +795,7 @@ mod tests { RESERVED_COL_NAME_POS, RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, }; - use crate::scan::{FileScanTask, FileScanTaskStream}; + use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Schema, SchemaRef, Type}; // INT96 encoding: [nanos_low_u32, nanos_high_u32, julian_day_u32] @@ -834,7 +836,8 @@ mod tests { .with_schema(schema) .with_project_field_ids(project_field_ids) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; reader @@ -1023,7 +1026,8 @@ mod tests { .with_project_field_ids(vec![1]) .with_case_sensitive(false) .with_key_metadata(Some(key_metadata)) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let batches: Vec = reader @@ -1095,7 +1099,8 @@ mod tests { .with_schema(schema) .with_project_field_ids(vec![1]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let result: Result, _> = @@ -1176,6 +1181,7 @@ mod tests { .with_data_sequence_number(data_sequence_number) .with_case_sensitive(false) .build() + .unwrap() } /// Asserts the logical per-row values of the `_last_updated_sequence_number` @@ -1419,7 +1425,8 @@ mod tests { .with_first_row_id(Some(100)) .with_data_sequence_number(Some(9)) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; @@ -1535,6 +1542,16 @@ mod tests { /// A scan task projecting `id` + `_row_id`, with the given `first_row_id`. fn row_id_task(file_path: String, first_row_id: Option) -> FileScanTask { + row_id_task_with_options(file_path, first_row_id, 0, 0, vec![]) + } + + fn row_id_task_with_options( + file_path: String, + first_row_id: Option, + start: u64, + length: u64, + deletes: Vec, + ) -> FileScanTask { let schema = Arc::new( Schema::builder() .with_schema_id(1) @@ -1547,15 +1564,17 @@ mod tests { FileScanTask::builder() .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) - .with_start(0) - .with_length(0) + .with_start(start) + .with_length(length) .with_data_file_path(file_path) .with_data_file_format(DataFileFormat::Parquet) .with_schema(schema) .with_project_field_ids(vec![1, RESERVED_FIELD_ID_ROW_ID]) .with_first_row_id(first_row_id) + .with_deletes(deletes) .with_case_sensitive(false) .build() + .unwrap() } /// Asserts the logical per-row values of the `_row_id` column across all batches, @@ -1650,24 +1669,24 @@ mod tests { let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path().to_str().unwrap(); - let mut meta_only = metadata_projection_task( + let meta_only = metadata_projection_task_with_first_row_id( write_parquet_with_wide_column(dir, "row_id_only.parquet", vec![], vec![]), id_and_wide_schema(), vec![RESERVED_FIELD_ID_ROW_ID], + Some(100), ); - meta_only.first_row_id = Some(100); let (batches, meta_only_bytes) = scan_task(meta_only).await; assert_eq!(batches[0].num_columns(), 1); assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]); // A scan that also projects the wide data column must read materially more. - let mut with_data = metadata_projection_task( + let with_data = metadata_projection_task_with_first_row_id( write_parquet_with_wide_column(dir, "row_id_only_ref.parquet", vec![], vec![]), id_and_wide_schema(), vec![2, RESERVED_FIELD_ID_ROW_ID], + Some(100), ); - with_data.first_row_id = Some(100); let (_, with_data_bytes) = scan_task(with_data).await; assert!( @@ -1686,23 +1705,23 @@ mod tests { let tmp_dir = TempDir::new().unwrap(); let dir = tmp_dir.path().to_str().unwrap(); - let mut meta_only = metadata_projection_task( + let meta_only = metadata_projection_task_with_first_row_id( write_parquet_with_wide_column(dir, "row_id_null.parquet", vec![], vec![]), id_and_wide_schema(), vec![RESERVED_FIELD_ID_ROW_ID], + None, ); - meta_only.first_row_id = None; let (batches, meta_only_bytes) = scan_task(meta_only).await; assert_eq!(batches[0].num_columns(), 1); assert_row_id_column(&batches, &[None, None, None]); - let mut with_data = metadata_projection_task( + let with_data = metadata_projection_task_with_first_row_id( write_parquet_with_wide_column(dir, "row_id_null_ref.parquet", vec![], vec![]), id_and_wide_schema(), vec![2, RESERVED_FIELD_ID_ROW_ID], + None, ); - with_data.first_row_id = None; let (_, with_data_bytes) = scan_task(with_data).await; assert!( @@ -1736,6 +1755,7 @@ mod tests { .with_data_sequence_number(Some(7)) .with_case_sensitive(false) .build() + .unwrap() }; let meta_only = lusn_only_task( @@ -1798,6 +1818,7 @@ mod tests { .with_unified_partition_type(Some(unified_type.clone())) .with_case_sensitive(false) .build() + .unwrap() }; let meta_only = partition_task( @@ -1910,7 +1931,8 @@ mod tests { .with_first_row_id(Some(100)) .with_data_sequence_number(Some(9)) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; @@ -1993,7 +2015,8 @@ mod tests { .with_project_field_ids(vec![1, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID]) .with_first_row_id(Some(100)) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; @@ -2160,6 +2183,7 @@ mod tests { .with_first_row_id(first_row_id) .with_case_sensitive(false) .build() + .unwrap() } #[tokio::test] @@ -2320,9 +2344,7 @@ mod tests { let start = 4 + metadata.row_group(0).compressed_size() as u64; let file_size = std::fs::metadata(&file_path).unwrap().len(); - let mut task = row_id_task(file_path, Some(100)); - task.start = start; - task.length = file_size - start; + let task = row_id_task_with_options(file_path, Some(100), start, file_size - start, vec![]); let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; @@ -2378,7 +2400,6 @@ mod tests { #[tokio::test] async fn test_row_id_survives_positional_delete() { - use crate::scan::FileScanTaskDeleteFile; use crate::spec::DataContentType; let tmp_dir = TempDir::new().unwrap(); @@ -2387,19 +2408,13 @@ mod tests { let data_path = write_plain_parquet(dir, "row_id_posdel_data.parquet", vec![], vec![]); let del_path = write_positional_delete(dir, "row_id_posdel.parquet", &data_path, &[1]); - let mut task = row_id_task(data_path, Some(100)); - task.deletes = vec![FileScanTaskDeleteFile { - file_path: del_path.clone(), - file_size_in_bytes: std::fs::metadata(&del_path).unwrap().len(), - file_type: DataContentType::PositionDeletes, - partition_spec_id: 0, - equality_ids: None, - referenced_data_file: None, - content_offset: None, - content_size_in_bytes: None, - record_count: None, - key_metadata: None, - }]; + let delete = FileScanTaskDeleteFile::builder() + .with_file_path(del_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_partition_spec_id(0) + .build(); + let task = row_id_task_with_options(data_path, Some(100), 0, 0, vec![delete]); let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; @@ -2508,7 +2523,8 @@ mod tests { .with_project_field_ids(vec![1]) .with_case_sensitive(false) .with_key_metadata(Some(wrong_key_metadata)) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let result: Result, _> = @@ -2597,7 +2613,8 @@ mod tests { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 2]) .with_case_sensitive(false) - .build()), + .build() + .unwrap()), Ok(FileScanTask::builder() .with_file_size_in_bytes( std::fs::metadata(format!("{table_location}/file_1.parquet")) @@ -2611,7 +2628,8 @@ mod tests { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 2]) .with_case_sensitive(false) - .build()), + .build() + .unwrap()), Ok(FileScanTask::builder() .with_file_size_in_bytes( std::fs::metadata(format!("{table_location}/file_2.parquet")) @@ -2625,7 +2643,8 @@ mod tests { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 2]) .with_case_sensitive(false) - .build()), + .build() + .unwrap()), ]; let tasks_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream; @@ -3111,6 +3130,15 @@ mod tests { file_path: String, schema: SchemaRef, project_field_ids: Vec, + ) -> FileScanTask { + metadata_projection_task_with_first_row_id(file_path, schema, project_field_ids, None) + } + + fn metadata_projection_task_with_first_row_id( + file_path: String, + schema: SchemaRef, + project_field_ids: Vec, + first_row_id: Option, ) -> FileScanTask { FileScanTask::builder() .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) @@ -3120,8 +3148,10 @@ mod tests { .with_data_file_format(DataFileFormat::Parquet) .with_schema(schema) .with_project_field_ids(project_field_ids) + .with_first_row_id(first_row_id) .with_case_sensitive(false) .build() + .unwrap() } /// Runs a single-task scan and returns the batches plus the bytes read from storage. @@ -3203,7 +3233,8 @@ mod tests { .with_project_field_ids(vec![RESERVED_FIELD_ID_POS]) .with_predicate(Some(bound)) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); // Row selection must be enabled for the predicate to filter rows. The row filter // reads `id` for its own evaluation even though `id` is not projected; the surviving @@ -3299,6 +3330,7 @@ mod tests { .with_data_sequence_number(Some(9)) .with_case_sensitive(false) .build() + .unwrap() }; let meta_only = seq_task(write("pos_seq.parquet"), vec![ @@ -3373,6 +3405,7 @@ mod tests { .with_data_sequence_number(Some(9)) .with_case_sensitive(false) .build() + .unwrap() }; let meta_only = seq_task(write("seq_only.parquet"), vec![ @@ -3437,6 +3470,7 @@ mod tests { .with_data_sequence_number(Some(9)) .with_case_sensitive(false) .build() + .unwrap() }; let meta_only = seq_task("seq_only_null_first.parquet", vec![ @@ -3510,7 +3544,7 @@ mod tests { #[tokio::test] async fn test_spec_id_only_reads_no_data_columns() { use crate::metadata_columns::{RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_SPEC_ID}; - use crate::spec::{PartitionSpec, Transform}; + use crate::spec::{Literal, PartitionSpec, Struct, Transform}; // `SELECT _spec_id` materializes an int constant from the task's partition spec id, // with no physical column to read. The RowNumber counter sizes it, so the scan @@ -3536,9 +3570,11 @@ mod tests { .with_data_file_format(DataFileFormat::Parquet) .with_schema(schema.clone()) .with_project_field_ids(project_field_ids) + .with_partition(Some(Struct::from_iter([Some(Literal::int(42))]))) .with_partition_spec(Some(spec.clone())) .with_case_sensitive(false) .build() + .unwrap() }; let meta_only = spec_id_task( diff --git a/crates/iceberg/src/arrow/reader/positional_deletes.rs b/crates/iceberg/src/arrow/reader/positional_deletes.rs index 900a3d46d6..a66d9daccf 100644 --- a/crates/iceberg/src/arrow/reader/positional_deletes.rs +++ b/crates/iceberg/src/arrow/reader/positional_deletes.rs @@ -453,7 +453,8 @@ mod tests { .build(), ]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let result = reader @@ -671,7 +672,8 @@ mod tests { .build(), ]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let result = reader @@ -883,7 +885,8 @@ mod tests { .build(), ]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let result = reader diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index f832a26521..873adb05ca 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -735,7 +735,8 @@ message schema { .with_schema(new_schema.clone()) .with_project_field_ids(vec![1, 2]) // Request both columns 'a' and 'b' .with_case_sensitive(false) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -834,7 +835,8 @@ message schema { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 2]) .with_case_sensitive(false) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -937,7 +939,8 @@ message schema { .with_project_field_ids(vec![2, 4]) .with_case_sensitive(false) .with_name_mapping(Some(name_mapping)) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -1045,7 +1048,8 @@ message schema { .with_case_sensitive(false) .with_name_mapping(Some(name_mapping)) .with_predicate(Some(predicate.bind(schema, true).unwrap())) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -1138,7 +1142,8 @@ message schema { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 3]) .with_case_sensitive(false) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -1225,7 +1230,8 @@ message schema { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 2, 3]) .with_case_sensitive(false) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -1326,7 +1332,8 @@ message schema { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 2]) .with_case_sensitive(false) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -1456,7 +1463,8 @@ message schema { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 2]) .with_case_sensitive(false) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -1553,7 +1561,8 @@ message schema { .with_schema(schema.clone()) .with_project_field_ids(vec![1, 5, 2]) .with_case_sensitive(false) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -1664,7 +1673,8 @@ message schema { .with_project_field_ids(vec![1, 2, 3]) .with_case_sensitive(false) .with_predicate(Some(predicate.bind(schema, true).unwrap())) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -1806,7 +1816,8 @@ message schema { .with_case_sensitive(false) .with_partition(Some(partition_data)) .with_partition_spec(Some(partition_spec)) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; @@ -2013,7 +2024,8 @@ message schema { .with_project_field_ids(vec![4]) .with_case_sensitive(false) .with_predicate(Some(predicate.bind(iceberg_schema, true).unwrap())) - .build())] + .build() + .unwrap())] .into_iter(), )) as FileScanTaskStream; diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 8c7b00aef3..2980392a4e 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -259,7 +259,8 @@ mod tests { .with_project_field_ids(vec![1]) .with_predicate(Some(predicate.bind(schema, true).unwrap())) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream }; @@ -551,7 +552,8 @@ mod tests { .with_project_field_ids(vec![1]) .with_record_count(Some(100)) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); // Task 2: read the second and third row groups let task2 = FileScanTask::builder() @@ -564,7 +566,8 @@ mod tests { .with_project_field_ids(vec![1]) .with_record_count(Some(200)) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let tasks1 = Box::pin(futures::stream::iter(vec![Ok(task1)])) as FileScanTaskStream; let result1 = reader @@ -699,7 +702,8 @@ mod tests { .with_schema(schema.clone()) .with_project_field_ids(vec![1]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let batches = reader @@ -806,7 +810,8 @@ mod tests { .with_schema(schema.clone()) .with_project_field_ids(vec![1]) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let batches = reader @@ -1133,26 +1138,18 @@ mod tests { .build(); let file_size = std::fs::metadata(&file_path).unwrap().len(); - let task = FileScanTask { - file_size_in_bytes: file_size, - start: 0, - length: 0, - record_count: None, - first_row_id: None, - data_sequence_number: None, - data_file_path: file_path.clone(), - data_file_format: DataFileFormat::Parquet, - schema: iceberg_schema.clone(), - project_field_ids: vec![1, 2], - predicate: Some(predicate), - deletes: vec![], - partition: None, - partition_spec: None, - name_mapping: None, - unified_partition_type: None, - case_sensitive: false, - key_metadata: None, - }; + let task = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(iceberg_schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_predicate(Some(predicate)) + .with_case_sensitive(false) + .build() + .unwrap(); let stream = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let batches: Vec = reader @@ -1228,19 +1225,16 @@ mod tests { .with_row_selection_enabled(true) .build(); - let task_sub2 = FileScanTask { - file_size_in_bytes: file_size, - start: 0, - length: 0, - record_count: None, - first_row_id: None, - data_sequence_number: None, - data_file_path: file_path.clone(), - data_file_format: DataFileFormat::Parquet, - schema: iceberg_schema.clone(), - project_field_ids: vec![1, 2], - predicate: Some(predicate_sub2), - deletes: vec![FileScanTaskDeleteFile { + let task_sub2 = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(iceberg_schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_predicate(Some(predicate_sub2)) + .with_deletes(vec![FileScanTaskDeleteFile { file_path: pos_del_path.clone(), file_type: DataContentType::PositionDeletes, partition_spec_id: 0, @@ -1251,14 +1245,10 @@ mod tests { content_size_in_bytes: None, record_count: None, key_metadata: None, - }], - partition: None, - partition_spec: None, - name_mapping: None, - unified_partition_type: None, - case_sensitive: false, - key_metadata: None, - }; + }]) + .with_case_sensitive(false) + .build() + .unwrap(); let stream_sub2 = Box::pin(futures::stream::iter(vec![Ok(task_sub2)])) as FileScanTaskStream; diff --git a/crates/iceberg/src/scan/context.rs b/crates/iceberg/src/scan/context.rs index 67b8cde4d8..6768fee0c1 100644 --- a/crates/iceberg/src/scan/context.rs +++ b/crates/iceberg/src/scan/context.rs @@ -128,7 +128,7 @@ impl ManifestEntryContext { ) .await; - Ok(FileScanTask::builder() + FileScanTask::builder() .with_file_size_in_bytes(self.manifest_entry.file_size_in_bytes()) .with_start(0) .with_length(self.manifest_entry.file_size_in_bytes()) @@ -150,7 +150,7 @@ impl ManifestEntryContext { .with_unified_partition_type(self.unified_partition_type.clone()) .with_case_sensitive(self.case_sensitive) .with_key_metadata(self.manifest_entry.data_file.key_metadata().map(Box::from)) - .build()) + .build() } } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 909edf40ba..2f0a876096 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -1961,8 +1961,7 @@ pub mod tests { assert!(!tasks.is_empty(), "expected at least one FileScanTask"); for task in &tasks { let mapping = task - .name_mapping - .as_ref() + .name_mapping() .expect("name_mapping should reach the FileScanTask"); assert_eq!(mapping.fields().len(), 1); assert_eq!(mapping.fields()[0].field_id(), Some(1)); @@ -2003,17 +2002,17 @@ pub mod tests { assert_eq!(tasks.len(), 2); - tasks.sort_by_key(|t| t.data_file_path.to_string()); + tasks.sort_by_key(|t| t.data_file_path().to_string()); // Check first task is added data file assert_eq!( - tasks[0].data_file_path, + tasks[0].data_file_path(), format!("{}/1.parquet", &fixture.table_location) ); // Check second task is existing data file assert_eq!( - tasks[1].data_file_path, + tasks[1].data_file_path(), format!("{}/3.parquet", &fixture.table_location) ); } @@ -2036,23 +2035,23 @@ pub mod tests { .unwrap(); assert_eq!(tasks.len(), 2); - tasks.sort_by_key(|task| task.data_file_path.to_string()); + tasks.sort_by_key(|task| task.data_file_path().to_string()); // The added file inherits the current snapshot's data sequence number, // the existing file keeps the one it was written with. assert_eq!( - tasks[0].data_file_path, + tasks[0].data_file_path(), format!("{}/1.parquet", &fixture.table_location) ); - assert_eq!(tasks[0].data_sequence_number, Some(1)); + assert_eq!(tasks[0].data_sequence_number(), Some(1)); assert_eq!( - tasks[1].data_file_path, + tasks[1].data_file_path(), format!("{}/3.parquet", &fixture.table_location) ); - assert_eq!(tasks[1].data_sequence_number, Some(0)); + assert_eq!(tasks[1].data_sequence_number(), Some(0)); // first_row_id is a v3 concept; a v2 manifest carries none. - assert!(tasks.iter().all(|task| task.first_row_id.is_none())); + assert!(tasks.iter().all(|task| task.first_row_id().is_none())); } #[tokio::test] @@ -2077,21 +2076,16 @@ pub mod tests { // The manifest-level first_row_id (42) is inherited onto the entry on // read, then carried onto the task. - assert_eq!(task.first_row_id, Some(42)); + assert_eq!(task.first_row_id(), Some(42)); // The data sequence number is threaded through the same v3 read path. - assert_eq!(task.data_sequence_number, Some(1)); + assert_eq!(task.data_sequence_number(), Some(1)); } #[tokio::test] - async fn test_filtered_scan_with_dropped_partition_source_column() { + async fn test_filtered_scan_rejects_dropped_partition_source_column() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // baseline: the same filtered scan against the table before evolution - let baseline = scan_y_gte_5(&fixture.table).await; - assert!(!baseline.is_empty()); - assert!(baseline.iter().all(|y| *y >= 5)); - // Evolve the table so that the manifests reference a historical spec whose source // column is no longer in the current schema: make an unpartitioned spec the // default, then drop the original spec's source column from the schema. @@ -2141,37 +2135,23 @@ pub mod tests { .metadata; let table = fixture.table.clone().with_metadata(Arc::new(metadata)); - // planning and reading must succeed, and the results must match the table before - // evolution: no rows wrongly pruned and none returned unfiltered - let evolved = scan_y_gte_5(&table).await; - assert_eq!(evolved, baseline); - } - - async fn scan_y_gte_5(table: &Table) -> Vec { let table_scan = table .scan() .select(["y"]) .with_filter(Reference::new("y").greater_than_or_equal_to(Datum::long(5))) .build() .unwrap(); - let batches: Vec<_> = table_scan - .to_arrow() + + let err = table_scan + .plan_files() .await .unwrap() - .try_collect() + .try_collect::>() .await - .unwrap(); + .unwrap_err(); - let mut values: Vec = batches - .iter() - .flat_map(|batch| { - let col = batch.column_by_name("y").unwrap(); - let arr = col.as_any().downcast_ref::().unwrap(); - (0..arr.len()).map(|i| arr.value(i)).collect::>() - }) - .collect(); - values.sort_unstable(); - values + assert_eq!(err.kind(), ErrorKind::Unexpected); + assert!(err.message().contains("No column with source column id 1")); } #[tokio::test] @@ -2655,14 +2635,7 @@ pub mod tests { let serialized = serde_json::to_string(&task).unwrap(); let deserialized: FileScanTask = serde_json::from_str(&serialized).unwrap(); - assert_eq!(task.data_file_path, deserialized.data_file_path); - assert_eq!(task.start, deserialized.start); - assert_eq!(task.length, deserialized.length); - assert_eq!(task.project_field_ids, deserialized.project_field_ids); - assert_eq!(task.predicate, deserialized.predicate); - assert_eq!(task.schema, deserialized.schema); - assert_eq!(task.first_row_id, deserialized.first_row_id); - assert_eq!(task.data_sequence_number, deserialized.data_sequence_number); + assert_eq!(task, deserialized); }; // without predicate @@ -2688,7 +2661,8 @@ pub mod tests { .with_data_sequence_number(Some(5)) .with_data_file_format(DataFileFormat::Parquet) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); test_fn(task); // with predicate @@ -2702,7 +2676,8 @@ pub mod tests { .with_schema(schema) .with_data_file_format(DataFileFormat::Avro) .with_case_sensitive(false) - .build(); + .build() + .unwrap(); test_fn(task); } @@ -3561,12 +3536,12 @@ pub mod tests { assert_eq!(tasks.len(), 1, "expected a single FileScanTask"); let task = &tasks[0]; assert!( - task.project_field_ids.contains(&RESERVED_FIELD_ID_POS), + task.project_field_ids().contains(&RESERVED_FIELD_ID_POS), "_pos field id must be projected into the FileScanTask" ); - assert_eq!(task.start, 0, "TableScan should plan whole-file tasks"); - assert_eq!(task.length, task.file_size_in_bytes); - assert!(task.deletes.is_empty()); + assert_eq!(task.start(), 0, "TableScan should plan whole-file tasks"); + assert_eq!(task.length(), task.file_size_in_bytes()); + assert!(task.deletes().is_empty()); // Reading that task yields absolute _pos 0..300 in order. let batches: Vec<_> = fixture @@ -3633,12 +3608,12 @@ pub mod tests { .unwrap(); assert_eq!(tasks.len(), 1); assert_eq!( - tasks[0].deletes.len(), + tasks[0].deletes().len(), 1, "positional delete file should be planned into the task" ); assert_eq!( - tasks[0].deletes[0].file_type, + tasks[0].deletes()[0].file_type, DataContentType::PositionDeletes ); diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 3b25bc3134..647fb3990b 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -21,12 +21,12 @@ use futures::stream::BoxStream; use serde::{Deserialize, Serialize, Serializer}; use typed_builder::TypedBuilder; -use crate::Result; use crate::expr::BoundPredicate; use crate::spec::{ DataContentType, DataFileFormat, ManifestEntryRef, NameMapping, PartitionSpec, Schema, SchemaRef, Struct, StructType, }; +use crate::{Error, ErrorKind, Result}; /// A stream of [`FileScanTask`]. pub type FileScanTaskStream = BoxStream<'static, Result>; @@ -51,21 +51,24 @@ where D: serde::Deserializer<'de> { /// A task to scan part of file. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TypedBuilder)] -#[builder(field_defaults(setter(prefix = "with_")))] +#[builder( + field_defaults(setter(prefix = "with_")), + build_method(into = Result) +)] pub struct FileScanTask { /// The total size of the data file in bytes, from the manifest entry. /// Used to skip a stat/HEAD request when reading Parquet footers. - pub file_size_in_bytes: u64, + file_size_in_bytes: u64, /// The start offset of the file to scan. - pub start: u64, + start: u64, /// The length of the file to scan. - pub length: u64, + length: u64, /// The number of records in the file to scan. /// /// This is an optional field, and only available if we are /// reading the entire data file. #[builder(default)] - pub record_count: Option, + record_count: Option, /// The first row id assigned to the data file. /// @@ -73,7 +76,7 @@ pub struct FileScanTask { /// explicit `_row_id`, it is this value plus the row's ordinal position. #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub first_row_id: Option, + first_row_id: Option, /// The data sequence number of the file, as opposed to its file sequence /// number: the sequence number preserved when a file is carried forward @@ -83,26 +86,26 @@ pub struct FileScanTask { /// Used to derive the `_last_updated_sequence_number` metadata column. #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub data_sequence_number: Option, + data_sequence_number: Option, /// The data file path corresponding to the task. - pub data_file_path: String, + data_file_path: String, /// The format of the file to scan. - pub data_file_format: DataFileFormat, + data_file_format: DataFileFormat, /// The schema of the file to scan. - pub schema: SchemaRef, + schema: SchemaRef, /// The field ids to project. - pub project_field_ids: Vec, + project_field_ids: Vec, /// The predicate to filter. #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub predicate: Option, + predicate: Option, /// The list of delete files that may need to be applied to this data file #[builder(default)] - pub deletes: Vec, + deletes: Vec, /// Partition data from the manifest entry, used to identify which columns can use /// constant values from partition metadata vs. reading from the data file. @@ -112,7 +115,7 @@ pub struct FileScanTask { #[serde(serialize_with = "serialize_not_implemented")] #[serde(deserialize_with = "deserialize_not_implemented")] #[builder(default)] - pub partition: Option, + partition: Option, /// The partition spec for this file, used to distinguish identity transforms /// (which use partition metadata constants) from non-identity transforms like @@ -122,7 +125,7 @@ pub struct FileScanTask { #[serde(serialize_with = "serialize_not_implemented")] #[serde(deserialize_with = "deserialize_not_implemented")] #[builder(default)] - pub partition_spec: Option>, + partition_spec: Option>, /// Name mapping from table metadata (property: schema.name-mapping.default), /// used to resolve field IDs from column names when Parquet files lack field IDs @@ -132,7 +135,7 @@ pub struct FileScanTask { #[serde(serialize_with = "serialize_not_implemented")] #[serde(deserialize_with = "deserialize_not_implemented")] #[builder(default)] - pub name_mapping: Option>, + name_mapping: Option>, /// The unified partition type across all specs in the table. /// When `RESERVED_FIELD_ID_PARTITION` is in the projected field IDs, the reader @@ -148,10 +151,10 @@ pub struct FileScanTask { #[serde(serialize_with = "serialize_not_implemented")] #[serde(deserialize_with = "deserialize_not_implemented")] #[builder(default)] - pub unified_partition_type: Option>, + unified_partition_type: Option>, /// Whether this scan task should treat column names as case-sensitive when binding predicates. - pub case_sensitive: bool, + case_sensitive: bool, /// Key metadata for encrypted data files (Parquet Modular Encryption). /// When present, the reader uses this to build `FileDecryptionProperties`. @@ -164,15 +167,60 @@ pub struct FileScanTask { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] - pub key_metadata: Option>, + key_metadata: Option>, } impl FileScanTask { + /// Returns the total size of the data file in bytes. + pub fn file_size_in_bytes(&self) -> u64 { + self.file_size_in_bytes + } + + /// Returns the start offset of the file to scan. + pub fn start(&self) -> u64 { + self.start + } + + /// Returns the length of the file to scan. + pub fn length(&self) -> u64 { + self.length + } + + /// Returns the number of records in the file when the whole file is scanned. + pub fn record_count(&self) -> Option { + self.record_count + } + + /// Returns the first row id assigned to the data file. + pub fn first_row_id(&self) -> Option { + self.first_row_id + } + + /// Returns the data sequence number of the file. + pub fn data_sequence_number(&self) -> Option { + self.data_sequence_number + } + /// Returns the data file path of this file scan task. pub fn data_file_path(&self) -> &str { &self.data_file_path } + /// Returns the format of the data file. + pub fn data_file_format(&self) -> DataFileFormat { + self.data_file_format + } + + /// Returns the schema of this file scan task as a reference. + pub fn schema(&self) -> &Schema { + &self.schema + } + + /// Returns the schema of this file scan task as a [`SchemaRef`]. + pub fn schema_ref(&self) -> SchemaRef { + self.schema.clone() + } + /// Returns the project field id of this file scan task. pub fn project_field_ids(&self) -> &[i32] { &self.project_field_ids @@ -183,14 +231,78 @@ impl FileScanTask { self.predicate.as_ref() } - /// Returns the schema of this file scan task as a reference - pub fn schema(&self) -> &Schema { - &self.schema + /// Returns the delete files that may need to be applied to the data file. + pub fn deletes(&self) -> &[FileScanTaskDeleteFile] { + &self.deletes } - /// Returns the schema of this file scan task as a SchemaRef - pub fn schema_ref(&self) -> SchemaRef { - self.schema.clone() + /// Returns the partition data from the manifest entry. + pub fn partition(&self) -> Option<&Struct> { + self.partition.as_ref() + } + + /// Returns the partition spec for the data file. + pub fn partition_spec(&self) -> Option<&Arc> { + self.partition_spec.as_ref() + } + + /// Returns the name mapping used to resolve field ids. + pub fn name_mapping(&self) -> Option<&Arc> { + self.name_mapping.as_ref() + } + + /// Returns the unified partition type across all table partition specs. + pub fn unified_partition_type(&self) -> Option<&Arc> { + self.unified_partition_type.as_ref() + } + + /// Returns whether names are treated as case-sensitive. + pub fn case_sensitive(&self) -> bool { + self.case_sensitive + } + + /// Returns the key metadata for the encrypted data file. + pub fn key_metadata(&self) -> Option<&[u8]> { + self.key_metadata.as_deref() + } + + fn validate(&self) -> Result<()> { + match (self.partition.as_ref(), self.partition_spec.as_deref()) { + (None, None) => Ok(()), + (None, Some(partition_spec)) if partition_spec.is_unpartitioned() => Ok(()), + (None, Some(_)) => Err(Error::new( + ErrorKind::DataInvalid, + "FileScanTask with a partitioned spec requires partition values", + )), + (Some(partition), None) if partition.fields().is_empty() => Ok(()), + (Some(_), None) => Err(Error::new( + ErrorKind::DataInvalid, + "Non-empty FileScanTask partition requires a partition spec", + )), + (Some(partition), Some(partition_spec)) + if partition.fields().len() != partition_spec.fields().len() => + { + Err(Error::new( + ErrorKind::DataInvalid, + format!( + "FileScanTask partition has {} fields but partition spec has {} fields", + partition.fields().len(), + partition_spec.fields().len() + ), + )) + } + (Some(_), Some(partition_spec)) => { + partition_spec.partition_type(&self.schema)?; + Ok(()) + } + } + } +} + +impl From for Result { + fn from(task: FileScanTask) -> Self { + task.validate()?; + Ok(task) } } @@ -282,3 +394,166 @@ pub struct FileScanTaskDeleteFile { #[builder(default)] pub key_metadata: Option>, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ErrorKind; + use crate::spec::{Literal, NestedField, PrimitiveType, Transform, Type}; + + fn build_file_scan_task( + schema: SchemaRef, + partition: Option, + partition_spec: Option>, + ) -> Result { + FileScanTask::builder() + .with_file_size_in_bytes(100) + .with_start(0) + .with_length(100) + .with_data_file_path("data_file_path".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![]) + .with_partition(partition) + .with_partition_spec(partition_spec) + .with_case_sensitive(false) + .build() + } + + fn schema_and_spec( + primitive_type: PrimitiveType, + transform: Transform, + ) -> (SchemaRef, Arc) { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::required( + 1, + "x", + Type::Primitive(primitive_type), + ))]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .add_partition_field("x", "x_partition", transform) + .unwrap() + .build() + .unwrap(), + ); + (schema, partition_spec) + } + + #[test] + fn test_file_scan_task_builder_rejects_non_empty_partition_without_spec() { + // Regression test for https://github.com/apache/iceberg-rust/issues/3130. + let err = build_file_scan_task( + Arc::new(Schema::builder().build().unwrap()), + Some(Struct::from_iter([Some(Literal::long(42))])), + None, + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "Non-empty FileScanTask partition requires a partition spec" + ); + } + + #[test] + fn test_file_scan_task_builder_accepts_empty_partition_without_spec() { + build_file_scan_task( + Arc::new(Schema::builder().build().unwrap()), + Some(Struct::empty()), + None, + ) + .unwrap(); + } + + #[test] + fn test_file_scan_task_builder_rejects_partitioned_spec_without_partition() { + let (schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity); + + let err = build_file_scan_task(schema, None, Some(partition_spec)).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "FileScanTask with a partitioned spec requires partition values" + ); + } + + #[test] + fn test_file_scan_task_builder_accepts_unpartitioned_spec_without_partition() { + build_file_scan_task( + Arc::new(Schema::builder().build().unwrap()), + None, + Some(Arc::new(PartitionSpec::unpartition_spec())), + ) + .unwrap(); + } + + #[test] + fn test_file_scan_task_builder_rejects_partition_arity_mismatch() { + let (schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity); + + let err = + build_file_scan_task(schema, Some(Struct::empty()), Some(partition_spec)).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("partition has 0 fields")); + assert!(err.message().contains("partition spec has 1 fields")); + } + + #[test] + fn test_file_scan_task_builder_rejects_dropped_partition_source_column() { + let (_historical_schema, partition_spec) = + schema_and_spec(PrimitiveType::Long, Transform::Identity); + let current_schema = Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::required( + 2, + "y", + Type::Primitive(PrimitiveType::String), + ))]) + .build() + .unwrap(), + ); + + let err = build_file_scan_task( + current_schema, + Some(Struct::from_iter([Some(Literal::long(42))])), + Some(partition_spec), + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::Unexpected); + assert!(err.message().contains("No column with source column id 1")); + } + + #[test] + fn test_file_scan_task_builder_rejects_partition_spec_incompatible_with_schema() { + let (_historical_schema, partition_spec) = + schema_and_spec(PrimitiveType::Timestamp, Transform::Day); + let current_schema = Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::required( + 1, + "x", + Type::Primitive(PrimitiveType::String), + ))]) + .build() + .unwrap(), + ); + + let err = build_file_scan_task( + current_schema, + Some(Struct::from_iter([Some(Literal::date(20_000))])), + Some(partition_spec), + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } +} diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 968a7b7236..55defaa5d7 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -1096,7 +1096,8 @@ mod tests { .with_project_field_ids(vec![1]) .with_case_sensitive(false) .with_key_metadata(data_file.key_metadata().map(Box::from)) - .build(); + .build() + .unwrap(); let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; let batches: Vec = reader .read(tasks) diff --git a/crates/integration_tests/tests/read_positional_deletes.rs b/crates/integration_tests/tests/read_positional_deletes.rs index 0f79596a12..fe8c9830d4 100644 --- a/crates/integration_tests/tests/read_positional_deletes.rs +++ b/crates/integration_tests/tests/read_positional_deletes.rs @@ -58,7 +58,7 @@ async fn test_read_table_with_positional_deletes() { // Scan plan phase should include delete files in file plan // when with_delete_file_processing_enabled == true - assert_eq!(plan[0].deletes.len(), 1); + assert_eq!(plan[0].deletes().len(), 1); // we should see two rows deleted, returning 10 rows instead of 12 let batch_stream = scan.to_arrow().await.unwrap(); From 31eb8dd4bc5a72e5cb20208b75e3ba52aa80ae26 Mon Sep 17 00:00:00 2001 From: Jannik Steinmann Date: Thu, 3 Sep 2026 01:18:58 +0200 Subject: [PATCH 22/74] refactor(auth) Emphasize AuthManagers aren't shared between catalogs (#3086) Take an owned AuthManager on catalog construction --- crates/catalog/rest/public-api.txt | 4 +-- crates/catalog/rest/src/auth/mod.rs | 8 +++-- crates/catalog/rest/src/catalog.rs | 54 ++++++++++++++++++----------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index d7e9d95e97..8afd581bcd 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -287,7 +287,7 @@ pub fn iceberg_catalog_rest::RestCatalog::update_namespace<'life0, 'life1, 'asyn pub fn iceberg_catalog_rest::RestCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub struct iceberg_catalog_rest::RestCatalogBuilder impl iceberg_catalog_rest::RestCatalogBuilder -pub fn iceberg_catalog_rest::RestCatalogBuilder::with_auth_manager(self, auth_manager: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_rest::RestCatalogBuilder::with_auth_manager(self, auth_manager: M) -> Self where M: iceberg_catalog_rest::AuthManager + 'static pub fn iceberg_catalog_rest::RestCatalogBuilder::with_client(self, client: reqwest::async_impl::client::Client) -> Self pub fn iceberg_catalog_rest::RestCatalogBuilder::with_session_context(self, context: iceberg::catalog::session::SessionContext) -> Self impl core::default::Default for iceberg_catalog_rest::RestCatalogBuilder @@ -322,7 +322,7 @@ pub fn iceberg_catalog_rest::RestSessionCatalog::update_table<'life0, 'life1, 'a pub struct iceberg_catalog_rest::RestSessionCatalogBuilder impl iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send -pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_auth_manager(self, auth_manager: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_auth_manager(self, auth_manager: M) -> Self where M: iceberg_catalog_rest::AuthManager + 'static pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_client(self, client: reqwest::async_impl::client::Client) -> Self pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs index e8bda50b91..b15dcdb480 100644 --- a/crates/catalog/rest/src/auth/mod.rs +++ b/crates/catalog/rest/src/auth/mod.rs @@ -38,11 +38,13 @@ pub const AUTH_TYPE_OAUTH2: &str = "oauth2"; /// Creates the [`AuthSession`]s used to authenticate REST catalog requests. /// -/// A manager is created once per catalog, either from the `rest.auth.type` -/// property or injected through +/// A manager is exclusively scoped to one catalog and must not be reused by +/// other catalogs. It is either created from the `rest.auth.type` property or +/// injected through /// [`RestCatalogBuilder::with_auth_manager`](crate::RestCatalogBuilder::with_auth_manager) or /// [`RestSessionCatalogBuilder::with_auth_manager`](crate::RestSessionCatalogBuilder::with_auth_manager). -/// It builds the sessions the catalog then keeps. +/// Catalog initialization calls [`AuthManager::catalog_session`] exactly once; +/// later sessions may rely on the state established by that call. /// /// Both methods are handed the catalog's [`HttpClient`], which an /// implementation may reuse for its own requests (e.g. a token exchange) so diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 841d5d33ef..aa36a074e7 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -128,8 +128,12 @@ impl RestCatalogBuilder { self } - /// Injects a custom auth manager, overriding the `rest.auth.type` configuration. - pub fn with_auth_manager(mut self, auth_manager: Arc) -> Self { + /// Sets a custom auth manager, overriding the `rest.auth.type` configuration. + /// + /// The builder takes ownership of the manager. The loaded catalog shares it + /// across authentication sessions and requests. + pub fn with_auth_manager(mut self, auth_manager: M) -> Self + where M: AuthManager + 'static { self.inner = self.inner.with_auth_manager(auth_manager); self } @@ -547,7 +551,7 @@ impl RestCatalog { fn new( context: SessionContext, config: RestCatalogConfig, - auth_manager: Option>, + auth_manager: Option>, storage_factory: Option>, runtime: Runtime, kms_client: Option>, @@ -706,13 +710,13 @@ impl RestSessionCatalog { /// Creates a `RestSessionCatalog` from a [`RestCatalogConfig`]. fn new( config: RestCatalogConfig, - auth_manager: Option>, + auth_manager: Option>, storage_factory: Option>, runtime: Runtime, kms_client: Option>, ) -> Self { Self { - auth_manager, + auth_manager: auth_manager.map(Arc::from), user_config: config, client: OnceCell::new(), storage_factory, @@ -1489,7 +1493,7 @@ impl SessionCatalog for RestSessionCatalog { #[derive(Debug)] pub struct RestSessionCatalogBuilder { config: RestCatalogConfig, - auth_manager: Option>, + auth_manager: Option>, storage_factory: Option>, kms_client_factory: Option>, runtime: Option, @@ -1521,9 +1525,13 @@ impl RestSessionCatalogBuilder { self } - /// Injects a custom auth manager, overriding the `rest.auth.type` configuration. - pub fn with_auth_manager(mut self, auth_manager: Arc) -> Self { - self.auth_manager = Some(auth_manager); + /// Sets a custom auth manager, overriding the `rest.auth.type` configuration. + /// + /// The builder takes ownership of the manager. The loaded catalog shares it + /// across authentication sessions and requests. + pub fn with_auth_manager(mut self, auth_manager: M) -> Self + where M: AuthManager + 'static { + self.auth_manager = Some(Box::new(auth_manager)); self } @@ -1676,14 +1684,18 @@ mod tests { use crate::request::HttpRequest; fn test_catalog(config: RestCatalogConfig) -> RestSessionCatalog { - test_catalog_with(config, None) + RestSessionCatalog::new(config, None, None, Runtime::current(), None) } - fn test_catalog_with( - config: RestCatalogConfig, - auth_manager: Option>, - ) -> RestSessionCatalog { - RestSessionCatalog::new(config, auth_manager, None, Runtime::current(), None) + fn test_catalog_with(config: RestCatalogConfig, auth_manager: M) -> RestSessionCatalog + where M: AuthManager + 'static { + RestSessionCatalog::new( + config, + Some(Box::new(auth_manager)), + None, + Runtime::current(), + None, + ) } fn test_client() -> HttpClient { @@ -2410,7 +2422,7 @@ mod tests { let catalog = RestCatalog::new( SessionContext::empty(), RestCatalogConfig::builder().uri(server.url()).build(), - Some(Arc::new(manager)), + Some(Box::new(manager)), Some(Arc::new(LocalFsStorageFactory)), Runtime::current(), None, @@ -2587,7 +2599,7 @@ mod tests { "tok-user".to_string(), )])) .build(), - Some(Arc::new(CapturingManager(captured.clone()))), + Some(Box::new(CapturingManager(captured.clone()))), Some(Arc::new(LocalFsStorageFactory)), Runtime::current(), None, @@ -2659,7 +2671,7 @@ mod tests { .uri(server.url()) .warehouse("client-wh".to_string()) .build(), - Some(Arc::new(CapturingManager(captured.clone()))), + Some(Box::new(CapturingManager(captured.clone()))), Some(Arc::new(LocalFsStorageFactory)), Runtime::current(), None, @@ -2691,7 +2703,7 @@ mod tests { .uri(server.url()) .warehouse("client-wh".to_string()) .build(), - Some(Arc::new(CapturingManager(captured.clone()))), + Some(Box::new(CapturingManager(captured.clone()))), Some(Arc::new(LocalFsStorageFactory)), Runtime::current(), None, @@ -2767,7 +2779,7 @@ mod tests { let catalog = RestCatalog::new( SessionContext::empty(), RestCatalogConfig::builder().uri(server.url()).build(), - Some(Arc::new(GuardManager(dropped.clone()))), + Some(Box::new(GuardManager(dropped.clone()))), Some(Arc::new(LocalFsStorageFactory)), Runtime::current(), None, @@ -2912,7 +2924,7 @@ mod tests { .build(); // The unknown auth type is never consulted. - let catalog = test_catalog_with(config, Some(Arc::new(StubAuthManager))); + let catalog = test_catalog_with(config, StubAuthManager); assert!(catalog.resolve_auth_manager().is_ok()); } From 107f449d198486258c0f2e459fc6a03183e4fad1 Mon Sep 17 00:00:00 2001 From: Kurtis Wright Date: Wed, 2 Sep 2026 16:40:53 -0700 Subject: [PATCH 23/74] fix(partition): Align DateTime Transform to Java's Implementation (#3022) * fix(partition): Align DateTime Transform to Java's Implementation * Add Doc Tests and Update Match Branching * Adding Spark Equivalence Tests * Update comments and return early * Removes requested integration tests --------- Co-authored-by: Kurtis Wright --- crates/iceberg/src/spec/partition.rs | 2 +- crates/iceberg/src/spec/transform.rs | 249 ++++++++++++++++++++++++-- crates/iceberg/src/spec/values/mod.rs | 2 +- 3 files changed, 241 insertions(+), 12 deletions(-) diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index 01c69080d8..0fe5739234 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -1748,7 +1748,7 @@ mod tests { assert_eq!( spec.partition_to_path(&data, schema.into()), - "id=42/name=alice/ts_hour=1000/empty_void=null" + "id=42/name=alice/ts_hour=1970-02-11-16/empty_void=null" ); } diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index 97ab638e79..af539b32a2 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -21,10 +21,12 @@ use std::cmp::Ordering; use std::fmt::{Display, Formatter}; use std::str::FromStr; +use chrono::Datelike; use fnv::FnvHashSet; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use super::values::decimal_utils::decimal_from_i128_with_scale; +use super::values::temporal::date; use super::{Datum, PrimitiveLiteral}; use crate::ErrorKind; use crate::error::{Error, Result}; @@ -36,6 +38,9 @@ use crate::spec::Literal; use crate::spec::datatypes::{PrimitiveType, Type}; use crate::transform::{BoxedTransformFunction, create_transform_function}; +/// The year the Unix epoch falls in, which every temporal ordinal counts from. +const UNIX_EPOCH_YEAR: i32 = 1970; + /// Transform is used to transform predicates to partition predicates, /// in addition to transforming data values. /// @@ -137,24 +142,138 @@ pub enum Transform { impl Transform { /// Returns a human-readable String representation of a transformed value. + /// + /// The temporal transforms store an ordinal count since the Unix epoch, and + /// this method renders that count as a date so that partition paths and + /// snapshot summary keys match the Java reference implementation: + /// + /// | Transform | Format | Example | + /// |-----------|-----------------|-----------------| + /// | `Year` | `yyyy` | `2017` | + /// | `Month` | `yyyy-MM` | `2017-06` | + /// | `Day` | `yyyy-MM-dd` | `2017-06-15` | + /// | `Hour` | `yyyy-MM-dd-HH` | `2017-06-15-16` | + /// + /// `Void` renders as `null`, as does an absent value for any transform. + /// + /// # Example + /// + /// ``` + /// use iceberg::spec::{Literal, PrimitiveType, Transform, Type}; + /// + /// let int = Type::Primitive(PrimitiveType::Int); + /// let date = Type::Primitive(PrimitiveType::Date); + /// + /// // A stored value carries no logical type of its own. For transforms that do + /// // not format it themselves, the declared field type decides how it renders. + /// let stored = Literal::int(17332); + /// assert_eq!( + /// Transform::Identity.to_human_string(&int, Some(&stored)), + /// "17332" + /// ); + /// assert_eq!( + /// Transform::Identity.to_human_string(&date, Some(&stored)), + /// "2017-06-15" + /// ); + /// + /// // The temporal transforms format their own ordinal and ignore the declared + /// // type. All four ordinals below are the same instant, 2017-06-15T16:00:00Z, + /// // counted at four granularities. + /// assert_eq!( + /// Transform::Year.to_human_string(&int, Some(&Literal::int(47))), + /// "2017" + /// ); + /// assert_eq!( + /// Transform::Month.to_human_string(&int, Some(&Literal::int(569))), + /// "2017-06" + /// ); + /// assert_eq!( + /// Transform::Day.to_human_string(&int, Some(&Literal::int(17332))), + /// "2017-06-15" + /// ); + /// assert_eq!( + /// Transform::Hour.to_human_string(&int, Some(&Literal::int(415984))), + /// "2017-06-15-16" + /// ); + /// + /// // `Void` and an absent value render as `null`. + /// assert_eq!( + /// Transform::Void.to_human_string(&int, Some(&Literal::int(47))), + /// "null" + /// ); + /// assert_eq!(Transform::Year.to_human_string(&int, None), "null"); + /// ``` pub fn to_human_string(&self, field_type: &Type, value: Option<&Literal>) -> String { - let Some(value) = value else { + let Some(value) = value.and_then(Literal::as_primitive_literal) else { return "null".to_string(); }; - if let Some(value) = value.as_primitive_literal() { - let field_type = field_type.as_primitive_type().unwrap(); - let datum = Datum::new(field_type.clone(), value); - - match self { - Self::Void => "null".to_string(), - _ => datum.to_human_string(), + match (*self, value) { + (Self::Void, _) => "null".to_string(), + // The temporal transforms store an ordinal count since the Unix epoch, + // which the datum arm below cannot render: it would show the raw count + // for `Year`, `Month` and `Hour`, and would leave `Day` dependent on the + // field type happening to be `date`. Java overrides `toHumanString` on + // each of these four transforms and ignores the declared type, so do the + // same. Any other literal falls through to the datum. + (Self::Year, PrimitiveLiteral::Int(ordinal)) => Self::human_year(ordinal), + (Self::Month, PrimitiveLiteral::Int(ordinal)) => Self::human_month(ordinal), + (Self::Day, PrimitiveLiteral::Int(ordinal)) => Self::human_day(ordinal), + (Self::Hour, PrimitiveLiteral::Int(ordinal)) => Self::human_hour(ordinal), + (_, value) => { + let field_type = field_type.as_primitive_type().unwrap(); + Datum::new(field_type.clone(), value).to_human_string() } - } else { - "null".to_string() } } + /// Formats a year ordinal, the number of years since 1970, as `yyyy`. + /// + /// Mirrors the output of `TransformUtil.humanYear`. + fn human_year(year_ordinal: i32) -> String { + format!("{:04}", UNIX_EPOCH_YEAR + year_ordinal) + } + + /// Formats a month ordinal, the number of months since 1970-01, as `yyyy-MM`. + /// + /// Mirrors the output of `TransformUtil.humanMonth`, which divides with + /// `Math.floorDiv` and `Math.floorMod` rather than `/` and `%`. Truncating + /// division rounds toward zero, which is the wrong direction before 1970: + /// ordinal -1 is 1969-12, but truncating yields 1970-01. `div_euclid` and + /// `rem_euclid` round toward negative infinity and so agree with Java. + fn human_month(month_ordinal: i32) -> String { + format!( + "{:04}-{:02}", + UNIX_EPOCH_YEAR + month_ordinal.div_euclid(12), + 1 + month_ordinal.rem_euclid(12) + ) + } + + /// Formats a day ordinal, the number of days since 1970-01-01, as `yyyy-MM-dd`. + /// + /// Mirrors the output of `TransformUtil.humanDay`. Like the Java `Days` + /// transform, whose signature is `toHumanString(Type alwaysDate, Integer + /// value)`, this ignores the declared field type rather than relying on it being + /// `date`. + fn human_day(day_ordinal: i32) -> String { + let date = date::days_to_date(day_ordinal); + format!("{:04}-{:02}-{:02}", date.year(), date.month(), date.day()) + } + + /// Formats an hour ordinal, the number of hours since 1970-01-01T00:00:00Z, + /// as `yyyy-MM-dd-HH`. + /// + /// Mirrors the output of `TransformUtil.humanHour`. `div_euclid` and `rem_euclid` + /// split the ordinal into whole days and the hour within the day so that hours + /// before 1970 round the way they do in Java. + fn human_hour(hour_ordinal: i32) -> String { + format!( + "{}-{:02}", + Self::human_day(hour_ordinal.div_euclid(24)), + hour_ordinal.rem_euclid(24) + ) + } + /// Get the return type of transform given the input type. /// Returns `None` if it can't be transformed. pub fn result_type(&self, input_type: &Type) -> Result { @@ -1119,4 +1238,114 @@ mod tests { check_boundary(PredicateOperator::GreaterThanOrEq, datum.clone(), datum); } } + + /// Renders `ordinal` through the public API with the given declared type. + fn human(transform: Transform, primitive: PrimitiveType, ordinal: i32) -> String { + transform.to_human_string(&Type::Primitive(primitive), Some(&Literal::int(ordinal))) + } + + /// Renders `ordinal` for a transform whose result type is `int`. + fn human_int(transform: Transform, ordinal: i32) -> String { + human(transform, PrimitiveType::Int, ordinal) + } + + #[test] + fn test_to_human_string_year() { + assert_eq!(human_int(Transform::Year, -1970), "0000"); + assert_eq!(human_int(Transform::Year, -1), "1969"); + assert_eq!(human_int(Transform::Year, 0), "1970"); + assert_eq!(human_int(Transform::Year, 47), "2017"); + } + + #[test] + fn test_to_human_string_month() { + assert_eq!(human_int(Transform::Month, -1970 * 12), "0000-01"); + assert_eq!(human_int(Transform::Month, -13), "1968-12"); + assert_eq!(human_int(Transform::Month, -12), "1969-01"); + assert_eq!(human_int(Transform::Month, -1), "1969-12"); + assert_eq!(human_int(Transform::Month, 0), "1970-01"); + assert_eq!(human_int(Transform::Month, 11), "1970-12"); + assert_eq!(human_int(Transform::Month, 12), "1971-01"); + assert_eq!(human_int(Transform::Month, 569), "2017-06"); + } + + #[test] + fn test_to_human_string_day() { + assert_eq!(human_int(Transform::Day, -1), "1969-12-31"); + assert_eq!(human_int(Transform::Day, 0), "1970-01-01"); + assert_eq!(human_int(Transform::Day, 31), "1970-02-01"); + assert_eq!(human_int(Transform::Day, 17332), "2017-06-15"); + } + + #[test] + fn test_to_human_string_hour() { + assert_eq!(human_int(Transform::Hour, -24), "1969-12-31-00"); + assert_eq!(human_int(Transform::Hour, -1), "1969-12-31-23"); + assert_eq!(human_int(Transform::Hour, 0), "1970-01-01-00"); + assert_eq!(human_int(Transform::Hour, 23), "1970-01-01-23"); + assert_eq!(human_int(Transform::Hour, 24), "1970-01-02-00"); + assert_eq!(human_int(Transform::Hour, 1000), "1970-02-11-16"); + assert_eq!(human_int(Transform::Hour, 415984), "2017-06-15-16"); + } + + /// The temporal transforms ignore the declared field type, matching the Java + /// signatures `toHumanString(Type alwaysInt, ..)` and + /// `toHumanString(Type alwaysDate, ..)`. + #[test] + fn test_to_human_string_ignores_declared_type_for_temporal_transforms() { + assert_eq!(human(Transform::Year, PrimitiveType::Date, 47), "2017"); + assert_eq!(human(Transform::Month, PrimitiveType::Date, 569), "2017-06"); + assert_eq!( + human(Transform::Day, PrimitiveType::Int, 17332), + "2017-06-15" + ); + assert_eq!( + human(Transform::Hour, PrimitiveType::Date, 415984), + "2017-06-15-16" + ); + } + + /// Transforms with no temporal format keep deferring to the datum, which + /// renders according to the declared field type. + #[test] + fn test_to_human_string_defers_to_datum_for_other_transforms() { + assert_eq!( + human(Transform::Identity, PrimitiveType::Int, 17332), + "17332" + ); + assert_eq!( + human(Transform::Identity, PrimitiveType::Date, 17332), + "2017-06-15" + ); + assert_eq!(human(Transform::Bucket(16), PrimitiveType::Int, 5), "5"); + + // A literal that is not an `int` also defers to the datum rather than being + // reported as `null`, so the temporal arms add no second behaviour change. + assert_eq!( + Transform::Year.to_human_string( + &Type::Primitive(PrimitiveType::String), + Some(&Literal::string("unformatted")) + ), + "unformatted" + ); + } + + #[test] + fn test_to_human_string_null_cases() { + assert_eq!(human(Transform::Void, PrimitiveType::Int, 47), "null"); + assert_eq!(human(Transform::Void, PrimitiveType::Date, 17332), "null"); + for transform in [ + Transform::Year, + Transform::Month, + Transform::Day, + Transform::Hour, + Transform::Identity, + Transform::Void, + ] { + assert_eq!( + transform.to_human_string(&Type::Primitive(PrimitiveType::Int), None), + "null" + ); + } + } } diff --git a/crates/iceberg/src/spec/values/mod.rs b/crates/iceberg/src/spec/values/mod.rs index 65374c74b7..d5dcb73aba 100644 --- a/crates/iceberg/src/spec/values/mod.rs +++ b/crates/iceberg/src/spec/values/mod.rs @@ -24,7 +24,7 @@ mod map; mod primitive; pub(crate) mod serde; mod struct_value; -mod temporal; +pub(crate) mod temporal; #[cfg(test)] mod tests; From dd4f32c7db9539e281f2a6cbdcb6564b70e88bd8 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 2 Sep 2026 20:30:39 -0400 Subject: [PATCH 24/74] feat(scan): [3/N] read and apply V3 deletion vectors (#3035) * feat(scan): carry deletion-vector coordinates on FileScanTaskDeleteFile Add referenced_data_file, content_offset, and content_size_in_bytes to FileScanTaskDeleteFile, populated from the delete file's manifest entry. These locate a deletion-vector blob and scope it to its data file, which the delete loader needs to read and apply V3 deletion vectors. Refs #2792. * fix after merging in main * fix public-api.txt * fix test name * update * apply V3 deletion vectors during scan * identify deletion vectors by content type and file format * Fix after merging upstream/main * Update comment wording. --- crates/iceberg/public-api.txt | 3 +- .../src/arrow/caching_delete_file_loader.rs | 481 ++++++++++++- .../iceberg/src/arrow/delete_file_loader.rs | 6 +- crates/iceberg/src/arrow/delete_filter.rs | 4 + .../src/arrow/reader/positional_deletes.rs | 189 +++++ crates/iceberg/src/arrow/reader/row_filter.rs | 1 + crates/iceberg/src/delete_file_index.rs | 655 ++++++++++++++++-- crates/iceberg/src/delete_vector.rs | 36 +- crates/iceberg/src/scan/context.rs | 2 +- crates/iceberg/src/scan/task.rs | 12 +- crates/iceberg/src/test_utils.rs | 15 + 11 files changed, 1296 insertions(+), 108 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 95df9e9443..57edfd6421 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1313,6 +1313,7 @@ pub struct iceberg::scan::FileScanTaskDeleteFile pub iceberg::scan::FileScanTaskDeleteFile::content_offset: core::option::Option pub iceberg::scan::FileScanTaskDeleteFile::content_size_in_bytes: core::option::Option pub iceberg::scan::FileScanTaskDeleteFile::equality_ids: core::option::Option> +pub iceberg::scan::FileScanTaskDeleteFile::file_format: iceberg::spec::DataFileFormat pub iceberg::scan::FileScanTaskDeleteFile::file_path: alloc::string::String pub iceberg::scan::FileScanTaskDeleteFile::file_size_in_bytes: u64 pub iceberg::scan::FileScanTaskDeleteFile::file_type: iceberg::spec::DataContentType @@ -1328,7 +1329,7 @@ impl core::fmt::Debug for iceberg::scan::FileScanTaskDeleteFile pub fn iceberg::scan::FileScanTaskDeleteFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::scan::FileScanTaskDeleteFile impl iceberg::scan::FileScanTaskDeleteFile -pub fn iceberg::scan::FileScanTaskDeleteFile::builder() -> FileScanTaskDeleteFileBuilder<((), (), (), (), (), (), (), (), (), ())> +pub fn iceberg::scan::FileScanTaskDeleteFile::builder() -> FileScanTaskDeleteFileBuilder<((), (), (), (), (), (), (), (), (), (), ())> impl serde_core::ser::Serialize for iceberg::scan::FileScanTaskDeleteFile pub fn iceberg::scan::FileScanTaskDeleteFile::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::scan::FileScanTaskDeleteFile diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 896248ff6d..905fa86d83 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -19,6 +19,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use arrow_array::{Array, ArrayRef, Int64Array, StringArray, StructArray}; +use bytes::Bytes; use futures::{StreamExt, TryStreamExt}; use tokio::sync::oneshot::{Receiver, channel}; @@ -27,15 +28,16 @@ use crate::arrow::delete_file_loader::BasicDeleteFileLoader; use crate::arrow::scan_metrics::ScanMetrics; use crate::arrow::{arrow_primitive_to_literal, arrow_schema_to_schema}; use crate::delete_vector::DeleteVector; +use crate::encryption::{EncryptedInputFile, StandardKeyMetadata}; use crate::expr::Predicate::AlwaysTrue; use crate::expr::{Predicate, Reference}; use crate::io::FileIO; use crate::runtime::Runtime; use crate::scan::{ArrowRecordBatchStream, FileScanTaskDeleteFile}; use crate::spec::{ - DataContentType, Datum, ListType, MapType, NestedField, NestedFieldRef, PartnerAccessor, - PrimitiveType, Schema, SchemaRef, SchemaWithPartnerVisitor, StructType, Type, VariantType, - visit_schema_with_partner, + DataContentType, DataFileFormat, Datum, ListType, MapType, NestedField, NestedFieldRef, + PartnerAccessor, PrimitiveType, Schema, SchemaRef, SchemaWithPartnerVisitor, StructType, Type, + VariantType, visit_schema_with_partner, }; use crate::{Error, ErrorKind, Result}; @@ -51,7 +53,6 @@ pub(crate) struct CachingDeleteFileLoader { // Intermediate context during processing of a delete file task. enum DeleteFileContext { - // TODO: Delete Vector loader from Puffin files ExistingEqDel, ExistingPosDel, PosDels { @@ -63,6 +64,15 @@ enum DeleteFileContext { equality_ids: HashSet, sender: tokio::sync::oneshot::Sender, }, + // A V3 deletion vector: the raw deletion-vector-v1 blob bytes, the data file whose rows it + // deletes, and the manifest's expected cardinality. The blob is decoded and validated in the + // parse phase. + DelVec { + data_file_path: String, + blob: Bytes, + record_count: u64, + dv_path: String, + }, } // Final result of the processing of a delete file task before @@ -72,6 +82,11 @@ enum ParsedDeleteFileContext { file_path: String, results: HashMap, }, + // A single deletion vector decoded from a Puffin blob, keyed by the data file it applies to. + DelVec { + data_file_path: String, + delete_vector: DeleteVector, + }, EqDel, ExistingPosDel, } @@ -117,12 +132,13 @@ impl CachingDeleteFileLoader { /// tasks from starting to load the same equality delete file. We spawn a task to load /// the EQ delete's record batch stream, convert it to a predicate, update the delete filter, /// and notify any task that was waiting for it. - /// * When this gets updated to add support for delete vectors, the load phase will return - /// a PuffinReader for them. + /// * For a V3 deletion vector, the load phase reads the blob's byte range directly from its + /// Puffin file (decrypting first if the entry carries key metadata), and the parse phase + /// decodes it into a single `DeleteVector`. /// * The parse phase parses each record batch stream according to its associated data type. /// The result of this is a map of data file paths to delete vectors for the positional - /// delete tasks (and in future for the delete vector tasks). For equality delete - /// file tasks, this results in an unbound Predicate. + /// delete tasks, or a single (data file path, delete vector) pair for a deletion vector + /// task. For equality delete file tasks, this results in an unbound Predicate. /// * The unbound Predicates resulting from equality deletes are sent to their associated oneshot /// channel to store them in the right place in the delete file managers state. /// * The results of all of these futures are awaited on in parallel with the specified @@ -143,10 +159,10 @@ impl CachingDeleteFileLoader { /// | /// | /// +-----------------------------+--------------------------+ - /// Pos Del Del Vec (Not yet Implemented) EQ Del + /// Pos Del Del Vec EQ Del /// | | | /// [parse pos del stream] [parse del vec puffin] [parse eq del] - /// HashMap HashMap (Predicate, Sender) + /// HashMap DeleteVector (Predicate, Sender) /// | | | /// | | [persist to state] /// | | () @@ -211,13 +227,22 @@ impl CachingDeleteFileLoader { .try_buffer_unordered(concurrency_limit_data_files); while let Some(item) = results_stream.next().await { - let item = item?; - if let ParsedDeleteFileContext::DelVecs { file_path, results } = item { - for (data_file_path, delete_vector) in results.into_iter() { + match item? { + ParsedDeleteFileContext::DelVecs { file_path, results } => { + for (data_file_path, delete_vector) in results.into_iter() { + del_filter.upsert_delete_vector(data_file_path, delete_vector); + } + // Mark the positional delete file as fully loaded so waiters can proceed + del_filter.finish_pos_del_load(&file_path); + } + ParsedDeleteFileContext::DelVec { + data_file_path, + delete_vector, + } => { del_filter.upsert_delete_vector(data_file_path, delete_vector); } - // Mark the positional delete file as fully loaded so waiters can proceed - del_filter.finish_pos_del_load(&file_path); + ParsedDeleteFileContext::EqDel + | ParsedDeleteFileContext::ExistingPosDel => {} } } @@ -239,6 +264,12 @@ impl CachingDeleteFileLoader { ) -> Result { match task.file_type { DataContentType::PositionDeletes => { + // A V3 deletion vector arrives as a PositionDeletes entry whose deletes live in + // a Puffin blob, not in a positional-delete parquet file. + if task.file_format == DataFileFormat::Puffin { + return Self::load_deletion_vector(task, basic_delete_file_loader).await; + } + match del_filter.try_start_pos_del_load(&task.file_path) { PosDelLoadAction::AlreadyLoaded => Ok(DeleteFileContext::ExistingPosDel), PosDelLoadAction::WaitFor(notified) => { @@ -299,6 +330,137 @@ impl CachingDeleteFileLoader { } } + /// Validates a deletion-vector task and returns what the read needs as typed values: + /// `(start, len, referenced data file path, expected cardinality)`. + /// + /// The spec requires `referenced_data_file`, `content_offset` and `content_size_in_bytes` on + /// a deletion vector, and a deletion vector is always built from a manifest entry, so it + /// always carries `record_count`. A missing one is a manifest-entry inconsistency rather + /// than an I/O failure. + /// + /// Equality and ordinary position deletes have no equivalent validation in this loader: a + /// malformed equality/position delete file fails loudly when the Parquet reader can't open + /// it. A deletion vector's coordinates instead drive a raw byte-range read with no format + /// to fail against, so a bad coordinate would otherwise decode silently into the wrong (or + /// no) deletes, per the same corrupted-blob concern Iceberg-Java validates in + /// `BitmapPositionDeleteIndex.deserializeBitmap`. + fn validate_deletion_vector_task( + task: &FileScanTaskDeleteFile, + ) -> Result<(u64, u64, String, u64)> { + let content_offset = task.content_offset.ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} is missing content_offset", + task.file_path + ), + ) + })?; + let content_size = task.content_size_in_bytes.ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} is missing content_size_in_bytes", + task.file_path + ), + ) + })?; + let data_file_path = task.referenced_data_file.clone().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} is missing referenced_data_file", + task.file_path + ), + ) + })?; + let record_count = task.record_count.ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("deletion vector {} is missing record_count", task.file_path), + ) + })?; + + let start = u64::try_from(content_offset).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} has negative content_offset {content_offset}", + task.file_path + ), + ) + })?; + let len = u64::try_from(content_size).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} has negative content_size_in_bytes {content_size}", + task.file_path + ), + ) + })?; + + Ok((start, len, data_file_path, record_count)) + } + + /// Validates a decoded deletion vector's cardinality against the manifest entry's + /// `record_count`, mirroring Iceberg-Java's `BitmapPositionDeleteIndex.deserializeBitmap`. + fn validate_deletion_vector_cardinality( + delete_vector: &DeleteVector, + expected: u64, + dv_path: &str, + ) -> Result<()> { + let actual = delete_vector.len(); + if actual != expected { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {dv_path} decoded to {actual} positions, expected {expected} from record_count" + ), + )); + } + Ok(()) + } + + /// Reads a V3 deletion vector blob directly from its Puffin file. + /// + /// The spec requires a delete manifest entry's `content_offset` / `content_size_in_bytes` to + /// match the blob's offset and length in the Puffin footer, so the blob is read by range + /// without parsing the footer. It is decoded into a [`DeleteVector`] in the parse phase. + /// + /// Decrypts the range read when `task.key_metadata` is set, the same way `ManifestReader` + /// decrypts a manifest file (`spec/manifest/reader.rs`): the coordinate space of + /// `content_offset` / `content_size_in_bytes` is the plaintext file, which is what + /// `EncryptedInputFile` reads over. + async fn load_deletion_vector( + task: &FileScanTaskDeleteFile, + basic_delete_file_loader: BasicDeleteFileLoader, + ) -> Result { + let (start, len, data_file_path, record_count) = Self::validate_deletion_vector_task(task)?; + + let input_file = basic_delete_file_loader + .file_io() + .new_input(&task.file_path)?; + let blob = match task.key_metadata.as_deref() { + Some(key_metadata) => { + let key_metadata = StandardKeyMetadata::decode(key_metadata)?; + EncryptedInputFile::new(input_file, key_metadata) + .reader() + .await? + .read(start..start + len) + .await? + } + None => input_file.reader().await?.read(start..start + len).await?, + }; + + Ok(DeleteFileContext::DelVec { + data_file_path, + blob, + record_count, + dv_path: task.file_path.clone(), + }) + } + async fn parse_file_content_for_task( ctx: DeleteFileContext, ) -> Result { @@ -312,6 +474,20 @@ impl CachingDeleteFileLoader { results: del_vecs, }) } + DeleteFileContext::DelVec { + data_file_path, + blob, + record_count, + dv_path, + } => { + let delete_vector = DeleteVector::deserialize(&blob)?; + Self::validate_deletion_vector_cardinality(&delete_vector, record_count, &dv_path)?; + + Ok(ParsedDeleteFileContext::DelVec { + data_file_path, + delete_vector, + }) + } DeleteFileContext::FreshEqDel { sender, batch_stream, @@ -700,6 +876,7 @@ mod tests { use crate::arrow::delete_filter::tests::setup; use crate::scan::FileScanTaskDeleteFile; use crate::spec::{DataContentType, Schema}; + use crate::test_utils::encode_dv_blob; #[tokio::test] async fn test_delete_file_loader_parse_equality_deletes() { @@ -1247,6 +1424,7 @@ mod tests { .with_file_path(pos_del_path.clone()) .with_file_size_in_bytes(std::fs::metadata(&pos_del_path).unwrap().len()) .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .build(); @@ -1254,6 +1432,7 @@ mod tests { .with_file_path(eq_delete_path.clone()) .with_file_size_in_bytes(std::fs::metadata(&eq_delete_path).unwrap().len()) .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .with_equality_ids(Some(vec![2, 3])) // Only use field IDs that exist in both schemas .build(); @@ -1384,4 +1563,276 @@ mod tests { // confirming that the second load reused the result from the first. assert!(Arc::ptr_eq(&dv1, &dv2)); } + + fn dv_task( + dv_path: String, + file_size: u64, + data_file_path: String, + content_offset: i64, + content_size: i64, + record_count: u64, + key_metadata: Option>, + ) -> FileScanTaskDeleteFile { + FileScanTaskDeleteFile::builder() + .with_file_path(dv_path) + .with_file_size_in_bytes(file_size) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some(data_file_path)) + .with_content_offset(Some(content_offset)) + .with_content_size_in_bytes(Some(content_size)) + .with_record_count(Some(record_count)) + .with_key_metadata(key_metadata) + .build() + } + + #[tokio::test] + async fn test_load_deletes_applies_deletion_vector() { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let blob = encode_dv_blob([0u64, 1, 5]); + + // Embed the blob in a Puffin-like file behind leading bytes so content_offset is + // non-zero, then let the loader read it back by range. + let content_offset = 12i64; + let content_size = blob.len() as i64; + let mut file_bytes = vec![0u8; content_offset as usize]; + file_bytes.extend_from_slice(&blob); + file_bytes.extend_from_slice(&[0u8; 8]); + let dv_path = format!("{table_location}/deletes.puffin"); + std::fs::write(&dv_path, &file_bytes).unwrap(); + + let data_file_path = format!("{table_location}/data-1.parquet"); + let dv = dv_task( + dv_path.clone(), + std::fs::metadata(&dv_path).unwrap().len(), + data_file_path.clone(), + content_offset, + content_size, + 3, + None, + ); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + + let loader = CachingDeleteFileLoader::new(file_io, 10, Runtime::current()); + let delete_filter = loader.load_deletes(&[dv], schema).await.unwrap().unwrap(); + + let delete_vector = delete_filter + .get_delete_vector_for_path(&data_file_path) + .expect("a delete vector should be indexed for the referenced data file"); + let mut positions: Vec = delete_vector.lock().unwrap().iter().collect(); + positions.sort_unstable(); + assert_eq!(positions, vec![0, 1, 5]); + } + + #[tokio::test] + async fn test_load_deletes_decrypts_deletion_vector() { + use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata}; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + let encoded_key_metadata = key_metadata.encode().unwrap(); + + let blob = encode_dv_blob([2u64, 4]); + let plaintext_size = blob.len() as i64; + let dv_path = format!("{table_location}/deletes.puffin"); + let output = EncryptedOutputFile::new(file_io.new_output(&dv_path).unwrap(), key_metadata); + output.write(Bytes::from(blob)).await.unwrap(); + + // content_offset / content_size_in_bytes are in the plaintext coordinate space, distinct + // from the ciphertext's on-disk size (header, nonce, and tag overhead). + let file_size = std::fs::metadata(&dv_path).unwrap().len(); + let data_file_path = format!("{table_location}/data-1.parquet"); + let dv = dv_task( + dv_path.clone(), + file_size, + data_file_path.clone(), + 0, + plaintext_size, + 2, + Some(encoded_key_metadata), + ); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + + let loader = CachingDeleteFileLoader::new(file_io, 10, Runtime::current()); + let delete_filter = loader.load_deletes(&[dv], schema).await.unwrap().unwrap(); + + let delete_vector = delete_filter + .get_delete_vector_for_path(&data_file_path) + .expect("a delete vector should be indexed for the referenced data file"); + let mut positions: Vec = delete_vector.lock().unwrap().iter().collect(); + positions.sort_unstable(); + assert_eq!(positions, vec![2, 4]); + } + + #[tokio::test] + async fn test_load_deletes_rejects_deletion_vector_cardinality_mismatch() { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let blob = encode_dv_blob([0u64, 1, 5]); + let dv_path = format!("{table_location}/deletes.puffin"); + std::fs::write(&dv_path, &blob).unwrap(); + + let data_file_path = format!("{table_location}/data-1.parquet"); + // record_count says 2 positions, but the blob decodes to 3. + let dv = dv_task( + dv_path.clone(), + std::fs::metadata(&dv_path).unwrap().len(), + data_file_path, + 0, + blob.len() as i64, + 2, + None, + ); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + + let loader = CachingDeleteFileLoader::new(file_io, 10, Runtime::current()); + let err = loader + .load_deletes(&[dv], schema) + .await + .unwrap() + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("expected 2 from record_count")); + } + + // A well-formed deletion-vector task, for tests that then clear or corrupt one field. + fn valid_dv_task() -> FileScanTaskDeleteFile { + dv_task( + "deletes.puffin".to_string(), + 100, + "data.parquet".to_string(), + 4, + 40, + 2, + None, + ) + } + + #[test] + fn test_validate_deletion_vector_task_rejects_missing_content_offset() { + let mut task = valid_dv_task(); + task.content_offset = None; + + let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("missing content_offset")); + } + + #[test] + fn test_validate_deletion_vector_task_rejects_missing_content_size() { + let mut task = valid_dv_task(); + task.content_size_in_bytes = None; + + let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("missing content_size_in_bytes")); + } + + #[test] + fn test_validate_deletion_vector_task_rejects_missing_referenced_data_file() { + let mut task = valid_dv_task(); + task.referenced_data_file = None; + + let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("missing referenced_data_file")); + } + + #[test] + fn test_validate_deletion_vector_task_rejects_missing_record_count() { + let mut task = valid_dv_task(); + task.record_count = None; + + let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("missing record_count")); + } + + #[test] + fn test_validate_deletion_vector_task_rejects_negative_content_offset() { + let mut task = valid_dv_task(); + task.content_offset = Some(-1); + + let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("negative content_offset")); + } + + #[test] + fn test_validate_deletion_vector_task_rejects_negative_content_size() { + let mut task = valid_dv_task(); + task.content_size_in_bytes = Some(-1); + + let err = CachingDeleteFileLoader::validate_deletion_vector_task(&task).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("negative content_size_in_bytes")); + } + + #[test] + fn test_validate_deletion_vector_task_accepts_valid_coordinates() { + let (start, len, data_file_path, record_count) = + CachingDeleteFileLoader::validate_deletion_vector_task(&valid_dv_task()).unwrap(); + assert_eq!(start, 4); + assert_eq!(len, 40); + assert_eq!(data_file_path, "data.parquet"); + assert_eq!(record_count, 2); + } + + #[test] + fn test_validate_deletion_vector_cardinality_accepts_matching_count() { + let mut dv = DeleteVector::default(); + dv.insert(1); + dv.insert(2); + + CachingDeleteFileLoader::validate_deletion_vector_cardinality(&dv, 2, "deletes.puffin") + .unwrap(); + } + + #[test] + fn test_validate_deletion_vector_cardinality_rejects_mismatched_count() { + let mut dv = DeleteVector::default(); + dv.insert(1); + + let err = + CachingDeleteFileLoader::validate_deletion_vector_cardinality(&dv, 2, "deletes.puffin") + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("expected 2 from record_count")); + } } diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index 13fa9740ae..5852cf1acd 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -182,7 +182,7 @@ mod tests { use crate::arrow::delete_filter::tests::create_pos_del_schema; use crate::encryption::StandardKeyMetadata; use crate::scan::FileScanTaskDeleteFile; - use crate::spec::DataContentType; + use crate::spec::{DataContentType, DataFileFormat}; let encryption_key = b"0123456789abcdef"; let aad_prefix = b"aad_prefix"; @@ -234,6 +234,7 @@ mod tests { file_path: del_path.clone(), file_size_in_bytes: std::fs::metadata(&del_path).unwrap().len(), file_type: DataContentType::PositionDeletes, + file_format: DataFileFormat::Parquet, partition_spec_id: 0, equality_ids: None, key_metadata: Some(Box::from(key_metadata.as_ref())), @@ -266,7 +267,7 @@ mod tests { use crate::encryption::StandardKeyMetadata; use crate::scan::FileScanTaskDeleteFile; - use crate::spec::DataContentType; + use crate::spec::{DataContentType, DataFileFormat}; let encryption_key = b"0123456789abcdef"; let aad_prefix = b"my-table-uuid!!"; @@ -312,6 +313,7 @@ mod tests { file_path: del_path.clone(), file_size_in_bytes: std::fs::metadata(&del_path).unwrap().len(), file_type: DataContentType::EqualityDeletes, + file_format: DataFileFormat::Parquet, partition_spec_id: 0, equality_ids: Some(vec![1]), key_metadata: Some(Box::from(key_metadata.as_ref())), diff --git a/crates/iceberg/src/arrow/delete_filter.rs b/crates/iceberg/src/arrow/delete_filter.rs index fb67d021e0..cef81afda8 100644 --- a/crates/iceberg/src/arrow/delete_filter.rs +++ b/crates/iceberg/src/arrow/delete_filter.rs @@ -441,6 +441,7 @@ pub(crate) mod tests { .len(), ) .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .build(); @@ -458,6 +459,7 @@ pub(crate) mod tests { .len(), ) .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .build(); @@ -475,6 +477,7 @@ pub(crate) mod tests { .len(), ) .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .build(); @@ -551,6 +554,7 @@ pub(crate) mod tests { .with_file_path("eq-del.parquet".to_string()) .with_file_size_in_bytes(1) // never read; this test fails before opening the file .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .build(), ]) diff --git a/crates/iceberg/src/arrow/reader/positional_deletes.rs b/crates/iceberg/src/arrow/reader/positional_deletes.rs index a66d9daccf..f0c7bf7f95 100644 --- a/crates/iceberg/src/arrow/reader/positional_deletes.rs +++ b/crates/iceberg/src/arrow/reader/positional_deletes.rs @@ -172,6 +172,7 @@ mod tests { use crate::io::FileIO; use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{DataContentType, DataFileFormat, NestedField, PrimitiveType, Schema, Type}; + use crate::test_utils::encode_dv_blob; fn build_test_row_group_meta( schema_descr: SchemaDescPtr, @@ -449,6 +450,7 @@ mod tests { .with_file_size_in_bytes(std::fs::metadata(&delete_file_path).unwrap().len()) .with_file_path(delete_file_path) .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .build(), ]) @@ -668,6 +670,7 @@ mod tests { .with_file_size_in_bytes(std::fs::metadata(&delete_file_path).unwrap().len()) .with_file_path(delete_file_path) .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .build(), ]) @@ -881,6 +884,7 @@ mod tests { .with_file_size_in_bytes(std::fs::metadata(&delete_file_path).unwrap().len()) .with_file_path(delete_file_path) .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) .build(), ]) @@ -926,4 +930,189 @@ mod tests { "Should have ids 101-200 (all of row group 1)" ); } + + /// End-to-end read of a data file with a V3 deletion vector applied. Exercises the whole + /// deletion-vector read path together: the DV coordinates on the scan task, the loader + /// reading the blob by range from its Puffin file, decoding it with DeleteVector::deserialize, + /// validating cardinality against record_count, and ArrowReader filtering the deleted rows. + #[tokio::test] + async fn test_deletion_vector_applied_end_to_end() { + use arrow_array::Int32Array; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + // Data file: ids 1..=5 at positions 0..=4. + let data_file_path = format!("{table_location}/data.parquet"); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(1..=5), + )]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&data_file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // Deletion vector deleting positions 1 and 3 (ids 2 and 4), serialized as a + // deletion-vector-v1 blob and embedded in a Puffin-like file at a non-zero offset. + let blob = encode_dv_blob([1u64, 3]); + let content_offset = 12i64; + let content_size = blob.len() as i64; + let mut dv_file_bytes = vec![0u8; content_offset as usize]; + dv_file_bytes.extend_from_slice(&blob); + let dv_path = format!("{table_location}/deletes.puffin"); + std::fs::write(&dv_path, &dv_file_bytes).unwrap(); + + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&data_file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_record_count(Some(5)) + .with_data_file_path(data_file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema.clone()) + .with_project_field_ids(vec![1]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path(dv_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&dv_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some(data_file_path.clone())) + .with_content_offset(Some(content_offset)) + .with_content_size_in_bytes(Some(content_size)) + .with_record_count(Some(2)) + .build(), + ]) + .with_case_sensitive(false) + .build(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let ids: Vec = result + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .iter() + .copied() + }) + .collect(); + + // Positions 1 and 3 (ids 2 and 4) are deleted; ids 1, 3, 5 remain. + assert_eq!(ids, vec![1, 3, 5]); + } + + /// A deletion vector whose decoded cardinality disagrees with the manifest entry's + /// `record_count` must fail the read rather than silently applying the wrong deletes, the + /// same invariant Iceberg-Java enforces in `BitmapPositionDeleteIndex.deserializeBitmap`. + #[tokio::test] + async fn test_deletion_vector_cardinality_mismatch_fails_read() { + use arrow_array::Int32Array; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let data_file_path = format!("{table_location}/data.parquet"); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(1..=5), + )]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&data_file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // The blob decodes to 2 positions, but the manifest entry claims 5. + let blob = encode_dv_blob([1u64, 3]); + let dv_path = format!("{table_location}/deletes.puffin"); + std::fs::write(&dv_path, &blob).unwrap(); + + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&data_file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_record_count(Some(5)) + .with_data_file_path(data_file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema.clone()) + .with_project_field_ids(vec![1]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path(dv_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&dv_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some(data_file_path.clone())) + .with_content_offset(Some(0)) + .with_content_size_in_bytes(Some(blob.len() as i64)) + .with_record_count(Some(5)) + .build(), + ]) + .with_case_sensitive(false) + .build(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::DataInvalid); + assert!(err.message().contains("expected 5 from record_count")); + } } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 2980392a4e..389231c461 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -1237,6 +1237,7 @@ mod tests { .with_deletes(vec![FileScanTaskDeleteFile { file_path: pos_del_path.clone(), file_type: DataContentType::PositionDeletes, + file_format: DataFileFormat::Parquet, partition_spec_id: 0, equality_ids: None, file_size_in_bytes: std::fs::metadata(&pos_del_path).unwrap().len(), diff --git a/crates/iceberg/src/delete_file_index.rs b/crates/iceberg/src/delete_file_index.rs index bab589385f..2e749cab1f 100644 --- a/crates/iceberg/src/delete_file_index.rs +++ b/crates/iceberg/src/delete_file_index.rs @@ -26,7 +26,8 @@ use tokio::sync::Notify; use crate::metadata_columns::RESERVED_FIELD_ID_DELETE_FILE_PATH; use crate::runtime::Runtime; use crate::scan::{DeleteFileContext, FileScanTaskDeleteFile}; -use crate::spec::{DataContentType, DataFile, PrimitiveLiteral, Struct}; +use crate::spec::{DataContentType, DataFile, DataFileFormat, PrimitiveLiteral, Struct}; +use crate::{Error, ErrorKind, Result}; /// Index of delete files #[derive(Debug, Clone)] @@ -36,8 +37,18 @@ pub(crate) struct DeleteFileIndex { #[derive(Debug)] enum DeleteFileIndexState { + // Arc, not Box: a waiter clones this out and awaits `notified_owned()` after dropping the + // read lock (a borrowed `notified()` future can't outlive the guard it was created under). + // If multiple callers arrive while still Populating, each clones its own handle to the same + // Notify, so one `notify_waiters()` call wakes all of them. Populating(Arc), - Populated(PopulatedDeleteFileIndex), + // Boxed because PopulatedDeleteFileIndex is large relative to the other variants; there is + // exactly one owner (this enum, behind the RwLock), so this needs heap indirection, not + // shared ownership. + Populated(Box), + // Boxed for the same reason: Error is large enough to trip the same size check, and is never + // cloned out of the lock, only read as `&Error` via deref. + Failed(Box), } #[derive(Debug)] @@ -46,7 +57,10 @@ struct PopulatedDeleteFileIndex { eq_deletes_by_partition: HashMap>>, pos_deletes_by_partition: HashMap>>, pos_deletes_by_path: HashMap>>, - // TODO: Deletion Vector support + // V3 deletion vectors, keyed by the data file they apply to (referenced_data_file). At most + // one exists per data file per snapshot, and when one applies it supersedes any position + // delete files for that data file, partition-scoped or path-scoped alike. + dvs_by_referenced_data_file: HashMap>, } /// Determines the single data file referenced by a position delete file, if any. @@ -79,6 +93,13 @@ fn referenced_data_file(data_file: &DataFile) -> Option { } } +/// Rebuilds an owned `Error` from the boxed `Error` cached in `DeleteFileIndexState::Failed`. +/// `Error` isn't `Clone`, so each waiter gets its own copy of the kind and message rather than +/// sharing the original's backtrace and source chain. +fn clone_failed_index_error(err: &Error) -> Error { + Error::new(err.kind(), err.message().to_string()) +} + impl DeleteFileIndex { /// create a new `DeleteFileIndex` along with the sender that populates it with delete files pub(crate) fn new(runtime: Runtime) -> (DeleteFileIndex, Sender) { @@ -96,11 +117,14 @@ impl DeleteFileIndex { let delete_files: Vec = delete_file_stream.collect::>().await; - let populated_delete_file_index = PopulatedDeleteFileIndex::new(delete_files); + let new_state = match PopulatedDeleteFileIndex::new(delete_files) { + Ok(index) => DeleteFileIndexState::Populated(Box::new(index)), + Err(err) => DeleteFileIndexState::Failed(Box::new(err)), + }; { let mut guard = state.write().unwrap(); - *guard = DeleteFileIndexState::Populated(populated_delete_file_index); + *guard = new_state; } notify.notify_waiters(); } @@ -110,11 +134,15 @@ impl DeleteFileIndex { } /// Gets all the delete files that apply to the specified data file. + /// + /// Fails if building the index found a spec violation, such as multiple deletion vectors + /// referencing the same data file, or if a matched deletion vector's sequence number + /// violates the spec relative to `seq_num`. pub(crate) async fn get_deletes_for_data_file( &self, data_file: &DataFile, seq_num: Option, - ) -> Vec { + ) -> Result> { // Create the `Notified` while holding the read lock. The read lock ensures that // when we go inside it, either the state is already at Populated or it is still // at Populating AND `notify_waiters()` has not been called yet. Any `Notified` @@ -127,6 +155,9 @@ impl DeleteFileIndex { DeleteFileIndexState::Populated(index) => { return index.get_deletes_for_data_file(data_file, seq_num); } + DeleteFileIndexState::Failed(err) => { + return Err(clone_failed_index_error(err)); + } } }; @@ -137,7 +168,10 @@ impl DeleteFileIndex { DeleteFileIndexState::Populated(index) => { index.get_deletes_for_data_file(data_file, seq_num) } - _ => unreachable!("Cannot be any other state than loaded"), + DeleteFileIndexState::Failed(err) => Err(clone_failed_index_error(err)), + DeleteFileIndexState::Populating(_) => { + unreachable!("Cannot still be Populating after being notified") + } } } } @@ -146,31 +180,83 @@ impl PopulatedDeleteFileIndex { /// Creates a new populated delete file index from a list of delete file contexts, which /// allows for fast lookup when determining which delete files apply to a given data file. /// - /// 1. Position deletes that reference a single data file, either through the + /// 1. A V3 deletion vector (a `PositionDeletes` entry stored as `Puffin`) is indexed by the + /// `referenced_data_file` field, which the spec requires for deletion vectors. + /// Fails if two deletion vectors reference the same data file: the spec allows at most + /// one deletion vector per data file per snapshot. + /// 2. Other position deletes that reference a single data file, either through the /// `referenced_data_file` field or through equal `file_path` column bounds, /// are indexed by that data file's path. - /// 2. All other position deletes are indexed by the partition extracted from + /// 3. All other position deletes are indexed by the partition extracted from /// their manifest entry. - /// 3. Equality deletes stored with an unpartitioned spec are applied as global + /// 4. Equality deletes stored with an unpartitioned spec are applied as global /// deletes, per the spec. All other equality deletes are indexed by partition. - fn new(files: Vec) -> PopulatedDeleteFileIndex { + fn new(files: Vec) -> Result { let mut eq_deletes_by_partition: HashMap>> = HashMap::default(); let mut pos_deletes_by_partition: HashMap>> = HashMap::default(); let mut pos_deletes_by_path: HashMap>> = HashMap::default(); + let mut dvs_by_referenced_data_file: HashMap> = + HashMap::default(); let mut global_equality_deletes: Vec> = vec![]; - files.into_iter().for_each(|ctx| { + for ctx in files { let arc_ctx = Arc::new(ctx); - let partition = arc_ctx.manifest_entry.data_file().partition(); + let data_file = arc_ctx.manifest_entry.data_file(); + let partition = data_file.partition(); match arc_ctx.manifest_entry.content_type() { DataContentType::PositionDeletes => { - if let Some(path) = referenced_data_file(arc_ctx.manifest_entry.data_file()) { + // A deletion vector is a position delete stored as a Puffin blob. The file + // format is what distinguishes it from a position delete parquet file. + if data_file.file_format() == DataFileFormat::Puffin { + // The spec requires referenced_data_file, content_offset and + // content_size_in_bytes on a deletion vector, so a missing one is a + // malformed manifest entry, not an ordinary position delete to fall back + // on. + let Some(path) = data_file.referenced_data_file() else { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} is missing referenced_data_file", + arc_ctx.manifest_entry.file_path() + ), + )); + }; + + if data_file.content_offset().is_none() + || data_file.content_size_in_bytes().is_none() + { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} is missing content_offset or content_size_in_bytes", + arc_ctx.manifest_entry.file_path() + ), + )); + } + + if let Some(existing) = + dvs_by_referenced_data_file.insert(path.clone(), arc_ctx) + { + let inserted = &dvs_by_referenced_data_file[&path]; + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "found multiple deletion vectors for data file {path}: {} and {}", + existing.manifest_entry.file_path(), + inserted.manifest_entry.file_path() + ), + )); + } + continue; + } + + if let Some(path) = referenced_data_file(data_file) { pos_deletes_by_path.entry(path).or_default().push(arc_ctx); } else { pos_deletes_by_partition @@ -192,22 +278,29 @@ impl PopulatedDeleteFileIndex { } _ => unreachable!(), } - }); + } - PopulatedDeleteFileIndex { + Ok(PopulatedDeleteFileIndex { global_equality_deletes, eq_deletes_by_partition, pos_deletes_by_partition, pos_deletes_by_path, - } + dvs_by_referenced_data_file, + }) } /// Determine all the delete files that apply to the provided `DataFile`. + /// + /// Fails if a matched deletion vector's partition or data sequence number is inconsistent + /// with the data file's: a data file's path is permanently tied to one partition, and the + /// spec guarantees a DV is only ever written at or after the sequence number of the data + /// file it applies to, so either violation means the delete manifest is inconsistent, not + /// that the DV simply doesn't apply. fn get_deletes_for_data_file( &self, data_file: &DataFile, seq_num: Option, - ) -> Vec { + ) -> Result> { let mut results = vec![]; self.global_equality_deletes @@ -233,6 +326,49 @@ impl PopulatedDeleteFileIndex { .for_each(|delete| results.push(delete.as_ref().into())); } + // A deletion vector supersedes all position delete files for its data file, per the spec: + // "readers ignore any position delete files that would otherwise match it, because the DV + // subsumes them". An exact path match on referenced_data_file is sufficient proof of + // applicability, the same as for pos_deletes_by_path below, so this is checked before + // (and instead of) pos_deletes_by_partition and pos_deletes_by_path. + if let Some(dv) = self.dvs_by_referenced_data_file.get(data_file.file_path()) { + let dv_data_file = dv.manifest_entry.data_file(); + // A file path belongs to exactly one partition for its lifetime, so an exact path + // match already implies partition equality; this checks that the manifest agrees, + // per the spec's explicit partition-equality condition for deletion vectors. + if data_file.partition() != dv_data_file.partition() + || data_file.partition_spec_id != dv.partition_spec_id + { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} references data file {} but its partition (spec {}, {:?}) does not match the data file's partition (spec {}, {:?})", + dv.manifest_entry.file_path(), + data_file.file_path(), + dv.partition_spec_id, + dv_data_file.partition(), + data_file.partition_spec_id, + data_file.partition() + ), + )); + } + + if let Some(seq_num) = seq_num { + let dv_seq = dv.manifest_entry.sequence_number(); + if dv_seq < Some(seq_num) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "deletion vector {} has data sequence number {dv_seq:?}, which must be >= the data file's sequence number {seq_num}", + dv.manifest_entry.file_path() + ), + )); + } + } + results.push(dv.as_ref().into()); + return Ok(results); + } + if let Some(deletes) = self.pos_deletes_by_partition.get(data_file.partition()) { deletes .iter() @@ -261,7 +397,7 @@ impl PopulatedDeleteFileIndex { .for_each(|delete| results.push(delete.as_ref().into())); } - results + Ok(results) } } @@ -297,23 +433,26 @@ mod tests { }) .collect(); - let delete_file_index = PopulatedDeleteFileIndex::new(delete_contexts); + let delete_file_index = PopulatedDeleteFileIndex::new(delete_contexts).unwrap(); let data_file = build_unpartitioned_data_file(); // All deletes apply to sequence 0 - let delete_files_to_apply_for_seq_0 = - delete_file_index.get_deletes_for_data_file(&data_file, Some(0)); + let delete_files_to_apply_for_seq_0 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap(); assert_eq!(delete_files_to_apply_for_seq_0.len(), 4); // All deletes apply to sequence 3 - let delete_files_to_apply_for_seq_3 = - delete_file_index.get_deletes_for_data_file(&data_file, Some(3)); + let delete_files_to_apply_for_seq_3 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(3)) + .unwrap(); assert_eq!(delete_files_to_apply_for_seq_3.len(), 4); // Last 3 deletes apply to sequence 4 - let delete_files_to_apply_for_seq_4 = - delete_file_index.get_deletes_for_data_file(&data_file, Some(4)); + let delete_files_to_apply_for_seq_4 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(4)) + .unwrap(); let actual_paths_to_apply_for_seq_4: Vec = delete_files_to_apply_for_seq_4 .into_iter() .map(|file| file.file_path) @@ -325,8 +464,9 @@ mod tests { ); // Last 3 deletes apply to sequence 5 - let delete_files_to_apply_for_seq_5 = - delete_file_index.get_deletes_for_data_file(&data_file, Some(5)); + let delete_files_to_apply_for_seq_5 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(5)) + .unwrap(); let actual_paths_to_apply_for_seq_5: Vec = delete_files_to_apply_for_seq_5 .into_iter() .map(|file| file.file_path) @@ -337,8 +477,9 @@ mod tests { ); // Only the last position delete applies to sequence 6 - let delete_files_to_apply_for_seq_6 = - delete_file_index.get_deletes_for_data_file(&data_file, Some(6)); + let delete_files_to_apply_for_seq_6 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(6)) + .unwrap(); let actual_paths_to_apply_for_seq_6: Vec = delete_files_to_apply_for_seq_6 .into_iter() .map(|file| file.file_path) @@ -352,8 +493,9 @@ mod tests { let partitioned_file = build_partitioned_data_file(&Struct::from_iter([Some(Literal::long(100))]), 1); - let delete_files_to_apply_for_partitioned_file = - delete_file_index.get_deletes_for_data_file(&partitioned_file, Some(0)); + let delete_files_to_apply_for_partitioned_file = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(0)) + .unwrap(); let actual_paths_to_apply_for_partitioned_file: Vec = delete_files_to_apply_for_partitioned_file .into_iter() @@ -389,24 +531,27 @@ mod tests { }) .collect(); - let delete_file_index = PopulatedDeleteFileIndex::new(delete_contexts); + let delete_file_index = PopulatedDeleteFileIndex::new(delete_contexts).unwrap(); let partitioned_file = build_partitioned_data_file(&Struct::from_iter([Some(Literal::long(100))]), spec_id); // All deletes apply to sequence 0 - let delete_files_to_apply_for_seq_0 = - delete_file_index.get_deletes_for_data_file(&partitioned_file, Some(0)); + let delete_files_to_apply_for_seq_0 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(0)) + .unwrap(); assert_eq!(delete_files_to_apply_for_seq_0.len(), 4); // All deletes apply to sequence 3 - let delete_files_to_apply_for_seq_3 = - delete_file_index.get_deletes_for_data_file(&partitioned_file, Some(3)); + let delete_files_to_apply_for_seq_3 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(3)) + .unwrap(); assert_eq!(delete_files_to_apply_for_seq_3.len(), 4); // Last 3 deletes apply to sequence 4 - let delete_files_to_apply_for_seq_4 = - delete_file_index.get_deletes_for_data_file(&partitioned_file, Some(4)); + let delete_files_to_apply_for_seq_4 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(4)) + .unwrap(); let actual_paths_to_apply_for_seq_4: Vec = delete_files_to_apply_for_seq_4 .into_iter() .map(|file| file.file_path) @@ -418,8 +563,9 @@ mod tests { ); // Last 3 deletes apply to sequence 5 - let delete_files_to_apply_for_seq_5 = - delete_file_index.get_deletes_for_data_file(&partitioned_file, Some(5)); + let delete_files_to_apply_for_seq_5 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(5)) + .unwrap(); let actual_paths_to_apply_for_seq_5: Vec = delete_files_to_apply_for_seq_5 .into_iter() .map(|file| file.file_path) @@ -430,8 +576,9 @@ mod tests { ); // Only the last position delete applies to sequence 6 - let delete_files_to_apply_for_seq_6 = - delete_file_index.get_deletes_for_data_file(&partitioned_file, Some(6)); + let delete_files_to_apply_for_seq_6 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(6)) + .unwrap(); let actual_paths_to_apply_for_seq_6: Vec = delete_files_to_apply_for_seq_6 .into_iter() .map(|file| file.file_path) @@ -444,8 +591,9 @@ mod tests { // Data file with different partition tuples does not match any delete files let partitioned_second_file = build_partitioned_data_file(&Struct::from_iter([Some(Literal::long(200))]), 1); - let delete_files_to_apply_for_different_partition = - delete_file_index.get_deletes_for_data_file(&partitioned_second_file, Some(0)); + let delete_files_to_apply_for_different_partition = delete_file_index + .get_deletes_for_data_file(&partitioned_second_file, Some(0)) + .unwrap(); let actual_paths_to_apply_for_different_partition: Vec = delete_files_to_apply_for_different_partition .into_iter() @@ -455,8 +603,9 @@ mod tests { // Data file with same tuple but different spec ID does not match any delete files let partitioned_different_spec = build_partitioned_data_file(&partition_one, 2); - let delete_files_to_apply_for_different_spec = - delete_file_index.get_deletes_for_data_file(&partitioned_different_spec, Some(0)); + let delete_files_to_apply_for_different_spec = delete_file_index + .get_deletes_for_data_file(&partitioned_different_spec, Some(0)) + .unwrap(); let actual_paths_to_apply_for_different_spec: Vec = delete_files_to_apply_for_different_spec .into_iter() @@ -474,15 +623,20 @@ mod tests { let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), partition_spec_id: 0, - }]); + }]) + .unwrap(); - let deletes_for_a = index.get_deletes_for_data_file(&data_file_a, Some(0)); + let deletes_for_a = index + .get_deletes_for_data_file(&data_file_a, Some(0)) + .unwrap(); assert_eq!(deletes_for_a.len(), 1); assert_eq!(deletes_for_a[0].file_path, pos_delete.file_path()); // The delete references data file A, so it must not apply to data file B // even though B shares A's partition. - let deletes_for_b = index.get_deletes_for_data_file(&data_file_b, Some(0)); + let deletes_for_b = index + .get_deletes_for_data_file(&data_file_b, Some(0)) + .unwrap(); assert!(deletes_for_b.is_empty()); } @@ -498,15 +652,20 @@ mod tests { let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), partition_spec_id: 0, - }]); + }]) + .unwrap(); assert_eq!( - index.get_deletes_for_data_file(&data_file_a, Some(0)).len(), + index + .get_deletes_for_data_file(&data_file_a, Some(0)) + .unwrap() + .len(), 1 ); assert!( index .get_deletes_for_data_file(&data_file_b, Some(0)) + .unwrap() .is_empty() ); } @@ -523,14 +682,21 @@ mod tests { let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), partition_spec_id: 0, - }]); + }]) + .unwrap(); assert_eq!( - index.get_deletes_for_data_file(&data_file_a, Some(0)).len(), + index + .get_deletes_for_data_file(&data_file_a, Some(0)) + .unwrap() + .len(), 1 ); assert_eq!( - index.get_deletes_for_data_file(&data_file_b, Some(0)).len(), + index + .get_deletes_for_data_file(&data_file_b, Some(0)) + .unwrap() + .len(), 1 ); } @@ -546,10 +712,14 @@ mod tests { let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), partition_spec_id: 0, - }]); + }]) + .unwrap(); assert_eq!( - index.get_deletes_for_data_file(&data_file, Some(0)).len(), + index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .len(), 1 ); } @@ -562,25 +732,39 @@ mod tests { let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), partition_spec_id: 0, - }]); + }]) + .unwrap(); // Position deletes apply when the delete's sequence number is greater // than or equal to the data file's. assert_eq!( - index.get_deletes_for_data_file(&data_file, Some(4)).len(), + index + .get_deletes_for_data_file(&data_file, Some(4)) + .unwrap() + .len(), 1 ); assert_eq!( - index.get_deletes_for_data_file(&data_file, Some(5)).len(), + index + .get_deletes_for_data_file(&data_file, Some(5)) + .unwrap() + .len(), 1 ); assert!( index .get_deletes_for_data_file(&data_file, Some(6)) + .unwrap() .is_empty() ); // Without a sequence number, the delete applies unconditionally. - assert_eq!(index.get_deletes_for_data_file(&data_file, None).len(), 1); + assert_eq!( + index + .get_deletes_for_data_file(&data_file, None) + .unwrap() + .len(), + 1 + ); } #[test] @@ -607,10 +791,12 @@ mod tests { manifest_entry: build_added_manifest_entry(5, &eq_delete).into(), partition_spec_id: 0, }, - ]); + ]) + .unwrap(); let mut actual_paths: Vec = index .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() .into_iter() .map(|delete| delete.file_path) .collect(); @@ -642,10 +828,12 @@ mod tests { manifest_entry: build_added_manifest_entry(6, &second_delete).into(), partition_spec_id: 0, }, - ]); + ]) + .unwrap(); let mut actual_paths: Vec = index .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() .into_iter() .map(|delete| delete.file_path) .collect(); @@ -679,15 +867,19 @@ mod tests { let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), partition_spec_id: 1, - }]); + }]) + .unwrap(); - let deletes_for_referenced = index.get_deletes_for_data_file(&data_file, Some(0)); + let deletes_for_referenced = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap(); assert_eq!(deletes_for_referenced.len(), 1); assert_eq!(deletes_for_referenced[0].file_path, pos_delete.file_path()); assert!( index .get_deletes_for_data_file(&same_partition_neighbor, Some(0)) + .unwrap() .is_empty() ); } @@ -706,15 +898,20 @@ mod tests { let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), partition_spec_id: 0, - }]); + }]) + .unwrap(); assert_eq!( - index.get_deletes_for_data_file(&data_file, Some(0)).len(), + index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .len(), 1 ); assert_eq!( index .get_deletes_for_data_file(&other_data_file, Some(0)) + .unwrap() .len(), 1 ); @@ -791,6 +988,330 @@ mod tests { ); } + #[test] + fn test_deletion_vector_supersedes_position_deletes() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_path = dv.file_path().to_string(); + let pos_del = build_partitioned_pos_delete(&partition, spec_id); + + let contexts = vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: spec_id, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_del).into(), + partition_spec_id: spec_id, + }, + ]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let applied: Vec = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .into_iter() + .map(|f| f.file_path) + .collect(); + + // Only the deletion vector applies; the partition-scoped position delete file, which + // would otherwise also match, is superseded. + assert_eq!(applied, vec![dv_path]); + } + + #[test] + fn test_deletion_vector_with_stale_sequence_number_is_rejected() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv = build_deletion_vector(data_file.file_path(), &partition, spec_id); + + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(3, &dv).into(), + partition_spec_id: spec_id, + }]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + // The DV's own sequence number (3) is less than the data file's (5): the spec guarantees + // a DV is never written before the data file it applies to, so this is an inconsistent + // manifest rather than a case where the DV simply doesn't apply. + let err = index + .get_deletes_for_data_file(&data_file, Some(5)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("must be >= the data file's sequence number") + ); + } + + #[test] + fn test_deletion_vector_with_mismatched_partition_is_rejected() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let other_partition = Struct::from_iter([Some(Literal::long(200))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + // Malformed: the DV's referenced_data_file matches data_file's path exactly, but the + // DV's own partition disagrees, a state that cannot arise from a valid writer since a + // file path is permanently tied to one partition. + let dv = build_deletion_vector(data_file.file_path(), &other_partition, spec_id); + + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: spec_id, + }]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let err = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("does not match the data file's partition") + ); + } + + #[test] + fn test_deletion_vector_with_mismatched_partition_spec_is_rejected() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let data_file = build_partitioned_data_file(&partition, 1); + + // Same partition value, but the DV's context was populated under a different partition + // spec id than the data file's: also a manifest inconsistency, since a file path is + // permanently tied to one partition spec. + let dv = build_deletion_vector(data_file.file_path(), &partition, 1); + + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: 2, + }]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let err = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("does not match the data file's partition") + ); + } + + #[test] + fn test_deletion_vector_supersedes_path_scoped_position_delete() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_path = dv.file_path().to_string(); + let pos_del = build_pos_delete_referencing(data_file.file_path()); + + let contexts = vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: spec_id, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_del).into(), + partition_spec_id: 0, + }, + ]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let applied: Vec = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .into_iter() + .map(|f| f.file_path) + .collect(); + + // Only the deletion vector applies; the path-scoped position delete file, which would + // otherwise also match by exact referenced_data_file path, is superseded. + assert_eq!(applied, vec![dv_path]); + } + + #[test] + fn test_deletion_vector_coexists_with_equality_delete() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_path = dv.file_path().to_string(); + let eq_del = build_unpartitioned_eq_delete(); + let eq_del_path = eq_del.file_path().to_string(); + + let contexts = vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: spec_id, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &eq_del).into(), + partition_spec_id: 0, + }, + ]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let mut applied: Vec = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .into_iter() + .map(|f| f.file_path) + .collect(); + applied.sort(); + + // A deletion vector only supersedes position deletes, not equality deletes: both apply. + let mut expected = vec![dv_path, eq_del_path]; + expected.sort(); + assert_eq!(applied, expected); + } + + #[test] + fn test_deletion_vector_missing_referenced_data_file_is_rejected() { + let malformed_dv = DataFileBuilder::default() + .file_path("deletes.puffin".to_string()) + .file_format(DataFileFormat::Puffin) + .content(DataContentType::PositionDeletes) + .record_count(1) + .content_offset(Some(4)) + .content_size_in_bytes(Some(40)) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(60) + .build() + .unwrap(); + + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &malformed_dv).into(), + partition_spec_id: 0, + }]; + + let err = PopulatedDeleteFileIndex::new(contexts).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("missing referenced_data_file")); + } + + #[test] + fn test_deletion_vector_missing_coordinates_is_rejected() { + let malformed_dv = DataFileBuilder::default() + .file_path("deletes.puffin".to_string()) + .file_format(DataFileFormat::Puffin) + .content(DataContentType::PositionDeletes) + .record_count(1) + .referenced_data_file(Some("data.parquet".to_string())) + .content_size_in_bytes(Some(40)) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(60) + .build() + .unwrap(); + + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &malformed_dv).into(), + partition_spec_id: 0, + }]; + + let err = PopulatedDeleteFileIndex::new(contexts).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("missing content_offset or content_size_in_bytes") + ); + } + + #[test] + fn test_multiple_deletion_vectors_for_same_data_file_is_rejected() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv_1 = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_2 = build_deletion_vector(data_file.file_path(), &partition, spec_id); + + let contexts = vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv_1).into(), + partition_spec_id: spec_id, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(6, &dv_2).into(), + partition_spec_id: spec_id, + }, + ]; + + let err = PopulatedDeleteFileIndex::new(contexts).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("multiple deletion vectors")); + } + + #[tokio::test] + async fn test_delete_file_index_propagates_multiple_dv_error_to_waiters() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv_1 = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_2 = build_deletion_vector(data_file.file_path(), &partition, spec_id); + + let (index, mut tx) = DeleteFileIndex::new(Runtime::current()); + tx.try_send(DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv_1).into(), + partition_spec_id: spec_id, + }) + .unwrap(); + tx.try_send(DeleteFileContext { + manifest_entry: build_added_manifest_entry(6, &dv_2).into(), + partition_spec_id: spec_id, + }) + .unwrap(); + drop(tx); + + let err = index + .get_deletes_for_data_file(&data_file, Some(0)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("multiple deletion vectors")); + + // A second caller, arriving after the index has already settled into the Failed state, + // must see the same error rather than panicking on an unexpected state. + let err = index + .get_deletes_for_data_file(&data_file, Some(0)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + // A V3 deletion vector: a PositionDeletes entry stored as a Puffin blob, scoped to a single + // data file via referenced_data_file. + fn build_deletion_vector( + referenced_data_file: &str, + partition: &Struct, + spec_id: i32, + ) -> DataFile { + DataFileBuilder::default() + .file_path(format!("{}-deletes.puffin", Uuid::new_v4())) + .file_format(DataFileFormat::Puffin) + .content(DataContentType::PositionDeletes) + .record_count(1) + .referenced_data_file(Some(referenced_data_file.to_string())) + .content_offset(Some(4)) + .content_size_in_bytes(Some(40)) + .partition(partition.clone()) + .partition_spec_id(spec_id) + .file_size_in_bytes(60) + .build() + .unwrap() + } + fn build_unpartitioned_eq_delete() -> DataFile { build_partitioned_eq_delete(&Struct::empty(), 0) } diff --git a/crates/iceberg/src/delete_vector.rs b/crates/iceberg/src/delete_vector.rs index e938ce5f69..4ab13d52b1 100644 --- a/crates/iceberg/src/delete_vector.rs +++ b/crates/iceberg/src/delete_vector.rs @@ -72,7 +72,6 @@ impl DeleteVector { Ok(positions.len()) } - #[allow(unused)] pub fn len(&self) -> u64 { self.inner.len() } @@ -91,17 +90,12 @@ impl DeleteVector { /// format: a directory of 32-bit key / 32-bit roaring bitmap pairs, ordered by unsigned /// comparison of the keys, one bitmap per key. /// - /// Cardinality is not checked here. The caller validates the decoded length against the - /// delete file's `record_count`, where the manifest metadata is available. - /// /// # Errors /// /// Returns [`ErrorKind::DataInvalid`] if the blob is shorter than the minimum, the length /// prefix or CRC does not match, the magic is wrong, the roaring bitmap count exceeds the /// portable format's maximum, the roaring directory's keys are not ordered by unsigned /// comparison, or the roaring payload fails to decode. - // Consumed by the scan delete loader once the deletion-vector read path is wired up. - #[allow(dead_code)] pub fn deserialize(blob: &[u8]) -> Result { if blob.len() < DV_MIN_BLOB_BYTES { return Err(Error::new( @@ -315,6 +309,22 @@ impl BitOrAssign for DeleteVector { } } +// Reproduces Iceberg-Java's `deletion-vector-v1` framing so tests can round-trip through +// `deserialize` without a Java writer, and so other test modules can build blob fixtures. +// Cross-implementation golden fixtures produced by Iceberg-Java are tracked separately; this +// only checks that our decode matches our encode. +#[cfg(test)] +pub(crate) fn frame_dv_blob(vector: &[u8]) -> Vec { + let body_len = DV_MAGIC_BYTES + vector.len(); + let mut blob = Vec::with_capacity(DV_LENGTH_PREFIX_BYTES + body_len + DV_CRC_BYTES); + blob.extend_from_slice(&(body_len as u32).to_be_bytes()); + blob.extend_from_slice(&DV_MAGIC); + blob.extend_from_slice(vector); + let crc = crc32fast::hash(&blob[DV_LENGTH_PREFIX_BYTES..]); + blob.extend_from_slice(&crc.to_be_bytes()); + blob +} + #[cfg(test)] mod tests { use super::*; @@ -372,20 +382,6 @@ mod tests { assert!(res.is_err()); } - // Reproduces Iceberg-Java's `deletion-vector-v1` framing so tests can round-trip through - // `deserialize` without a Java writer. Cross-implementation golden fixtures produced by - // Iceberg-Java are tracked separately; this only checks that our decode matches our encode. - fn frame_dv_blob(vector: &[u8]) -> Vec { - let body_len = DV_MAGIC_BYTES + vector.len(); - let mut blob = Vec::with_capacity(DV_LENGTH_PREFIX_BYTES + body_len + DV_CRC_BYTES); - blob.extend_from_slice(&(body_len as u32).to_be_bytes()); - blob.extend_from_slice(&DV_MAGIC); - blob.extend_from_slice(vector); - let crc = crc32fast::hash(&blob[DV_LENGTH_PREFIX_BYTES..]); - blob.extend_from_slice(&crc.to_be_bytes()); - blob - } - fn encode_dv_blob(dv: &DeleteVector) -> Vec { let mut vector = Vec::with_capacity(dv.inner.serialized_size()); dv.inner.serialize_into(&mut vector).unwrap(); diff --git a/crates/iceberg/src/scan/context.rs b/crates/iceberg/src/scan/context.rs index 6768fee0c1..a598975b96 100644 --- a/crates/iceberg/src/scan/context.rs +++ b/crates/iceberg/src/scan/context.rs @@ -126,7 +126,7 @@ impl ManifestEntryContext { self.manifest_entry.data_file(), self.manifest_entry.sequence_number(), ) - .await; + .await?; FileScanTask::builder() .with_file_size_in_bytes(self.manifest_entry.file_size_in_bytes()) diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 647fb3990b..3aff0f4ae6 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -318,6 +318,7 @@ impl From<&DeleteFileContext> for FileScanTaskDeleteFile { .with_file_path(ctx.manifest_entry.file_path().to_string()) .with_file_size_in_bytes(ctx.manifest_entry.file_size_in_bytes()) .with_file_type(ctx.manifest_entry.content_type()) + .with_file_format(ctx.manifest_entry.data_file().file_format()) .with_partition_spec_id(ctx.partition_spec_id) .with_equality_ids(ctx.manifest_entry.data_file.equality_ids.clone()) .with_referenced_data_file(ctx.manifest_entry.data_file.referenced_data_file.clone()) @@ -348,6 +349,10 @@ pub struct FileScanTaskDeleteFile { /// delete file type pub file_type: DataContentType, + /// The delete file's format, from the manifest entry. A `PositionDeletes` entry written as + /// `Puffin` is a V3 deletion vector; one written as `Parquet` is a position delete file. + pub file_format: DataFileFormat, + /// partition id pub partition_spec_id: i32, @@ -383,8 +388,11 @@ pub struct FileScanTaskDeleteFile { #[builder(default)] pub record_count: Option, - /// Key metadata for encrypted delete files (Parquet Modular Encryption). - /// When present, the reader uses this to build `FileDecryptionProperties`. + /// Key metadata for an encrypted delete file. When present, the reader uses this to + /// decrypt the file: for a Parquet equality or position delete file, this builds + /// `FileDecryptionProperties` (Parquet Modular Encryption); for a deletion vector, whose + /// Puffin file has no native encryption, this wraps the range read in an + /// `EncryptedInputFile` (AGS1 stream encryption). /// /// Same plaintext-DEK trust boundary as [`FileScanTask::key_metadata`]: /// this is serialized into the scan plan and crosses the planner -> worker diff --git a/crates/iceberg/src/test_utils.rs b/crates/iceberg/src/test_utils.rs index 63f40111f8..e80a51d98d 100644 --- a/crates/iceberg/src/test_utils.rs +++ b/crates/iceberg/src/test_utils.rs @@ -24,6 +24,8 @@ use std::sync::{Arc, OnceLock}; use arrow_array::RecordBatch; use expect_test::Expect; use itertools::Itertools; +#[cfg(test)] +use roaring::RoaringTreemap; use crate::TableIdent; #[cfg(test)] @@ -120,6 +122,19 @@ pub(crate) fn make_encryption_manager(table_key_id: &str) -> Arc) -> Vec { + let mut bitmap = RoaringTreemap::new(); + for pos in positions { + bitmap.insert(pos); + } + let mut vector = Vec::new(); + bitmap.serialize_into(&mut vector).unwrap(); + crate::delete_vector::frame_dv_blob(&vector) +} + /// Build a table backed by the V3 encryption fixture and an in-memory KMS, /// so it has an [`EncryptionManager`](crate::encryption::EncryptionManager). /// From c8fc70e5dcf4591a4c0d834223b4c7078ddb28c2 Mon Sep 17 00:00:00 2001 From: Daniel Carl Jones Date: Thu, 3 Sep 2026 17:00:40 +0100 Subject: [PATCH 25/74] fix: Address merge conflicts raising Clippy issues (#3137) --- crates/iceberg/src/arrow/reader/pipeline.rs | 1 + crates/iceberg/src/arrow/reader/positional_deletes.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 894e51d76d..5354c72043 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -2410,6 +2410,7 @@ mod tests { let delete = FileScanTaskDeleteFile::builder() .with_file_path(del_path.clone()) + .with_file_format(DataFileFormat::Parquet) .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) .with_file_type(DataContentType::PositionDeletes) .with_partition_spec_id(0) diff --git a/crates/iceberg/src/arrow/reader/positional_deletes.rs b/crates/iceberg/src/arrow/reader/positional_deletes.rs index f0c7bf7f95..2915596da7 100644 --- a/crates/iceberg/src/arrow/reader/positional_deletes.rs +++ b/crates/iceberg/src/arrow/reader/positional_deletes.rs @@ -1010,7 +1010,7 @@ mod tests { .with_case_sensitive(false) .build(); - let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let tasks = Box::pin(futures::stream::iter(vec![task])) as FileScanTaskStream; let result = reader .read(tasks) .unwrap() @@ -1107,7 +1107,7 @@ mod tests { .with_case_sensitive(false) .build(); - let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let tasks = Box::pin(futures::stream::iter(vec![task])) as FileScanTaskStream; let result: Result, _> = reader.read(tasks).unwrap().stream().try_collect().await; From 665c64e48e8d33797ecb1a421f327edd9b024879 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Thu, 3 Sep 2026 13:28:56 -0400 Subject: [PATCH 26/74] deps: upgrade to DataFusion 55.0 and Arrow/Parquet 59.2 (#2958) * bump df to 55.0 (head on main) and arrow to 59.1 * upmerge main after opendal bump * fix some CI failures * python version bumps * bump to branch-55 and rc2 commit. bump to arrow and parquet 59.2 * bump to branch-55 and rc2 commit. bump to arrow and parquet 59.2 * bump to final crates, fix API changes * update public API * gate test --- Cargo.lock | 519 ++++++++++-------- Cargo.toml | 30 +- bindings/python/pyproject.toml | 2 +- .../python/src/datafusion_table_provider.rs | 8 +- .../tests/test_datafusion_table_provider.py | 4 +- .../python/tests/test_huggingface_and_cdc.py | 11 +- bindings/python/uv.lock | 2 +- crates/iceberg/Cargo.toml | 2 +- crates/integrations/datafusion/public-api.txt | 29 +- .../datafusion/src/physical_plan/commit.rs | 17 +- .../src/physical_plan/metadata_scan.rs | 10 +- .../datafusion/src/physical_plan/scan.rs | 10 +- .../datafusion/src/physical_plan/write.rs | 17 +- .../src/table/table_provider_factory.rs | 22 +- 14 files changed, 395 insertions(+), 288 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ebec585a6..84a87465f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,7 +165,7 @@ dependencies = [ "digest 0.10.7", "log", "miniz_oxide", - "num-bigint", + "num-bigint 0.4.8", "quad-rand", "rand 0.9.5", "regex-lite", @@ -227,9 +227,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" dependencies = [ "arrow-arith", "arrow-array", @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -263,9 +263,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" dependencies = [ "ahash", "arrow-buffer", @@ -275,6 +275,7 @@ dependencies = [ "chrono-tz", "half", "hashbrown 0.17.1", + "libc", "num-complex", "num-integer", "num-traits", @@ -282,9 +283,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" dependencies = [ "arrow-array", "arrow-buffer", @@ -306,21 +307,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" dependencies = [ "arrow-array", "arrow-buffer", @@ -329,7 +330,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64 0.23.0", "chrono", "comfy-table", "half", @@ -340,9 +341,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" dependencies = [ "arrow-array", "arrow-cast", @@ -355,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -368,9 +369,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" dependencies = [ "arrow-array", "arrow-buffer", @@ -378,15 +379,15 @@ dependencies = [ "arrow-schema", "arrow-select", "flatbuffers", - "lz4_flex", + "lz4_flex 0.14.0", "zstd", ] [[package]] name = "arrow-json" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -409,9 +410,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" dependencies = [ "arrow-array", "arrow-buffer", @@ -422,9 +423,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" +checksum = "c196ecc25b3a8dcbc1d842f2619cee653dcfa2fb8b56a291bc0481c3cf5c3821" dependencies = [ "arrow-array", "arrow-data", @@ -434,9 +435,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -447,9 +448,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -458,9 +459,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" dependencies = [ "ahash", "arrow-array", @@ -472,9 +473,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" dependencies = [ "arrow-array", "arrow-buffer", @@ -552,7 +553,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -563,7 +564,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -994,7 +995,7 @@ checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1116,7 +1117,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", "serde", @@ -1225,7 +1226,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.119", ] [[package]] @@ -1292,7 +1293,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1444,7 +1445,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1849,7 +1850,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] @@ -1862,7 +1863,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] @@ -1873,7 +1874,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1884,7 +1885,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1903,9 +1904,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" +checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4" dependencies = [ "arrow", "arrow-schema", @@ -1941,7 +1942,7 @@ dependencies = [ "flate2", "futures", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1957,9 +1958,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f" dependencies = [ "arrow", "async-trait", @@ -1973,7 +1974,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1982,9 +1983,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" +checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4" dependencies = [ "arrow", "async-trait", @@ -1998,16 +1999,17 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-cli" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f652cbc0854d1e24c8038e2c7be53ef395768b8713fceaaa00789ae6134a5ee" +checksum = "5d8b87ff78e243da48b836ced274799a1799b4a2fd321700e4b8f3426c01aa7e" dependencies = [ "arrow", "async-trait", @@ -2033,9 +2035,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6" dependencies = [ "arrow", "arrow-ipc", @@ -2046,9 +2048,10 @@ dependencies = [ "hashbrown 0.17.1", "hex", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -2060,9 +2063,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea" dependencies = [ "futures", "log", @@ -2071,9 +2074,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" +checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b" dependencies = [ "arrow", "async-compression", @@ -2089,11 +2092,12 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -2107,9 +2111,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678" dependencies = [ "arrow", "arrow-ipc", @@ -2122,18 +2126,19 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-avro" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" +checksum = "c1c9587cff8163f9bfcb186e8664ba85cb327031b8ff14330307f961ea2fc196" dependencies = [ "arrow", "arrow-avro", @@ -2150,9 +2155,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d" dependencies = [ "arrow", "async-trait", @@ -2164,6 +2169,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -2173,9 +2179,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" +checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2" dependencies = [ "arrow", "async-trait", @@ -2187,6 +2193,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -2196,11 +2203,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" +checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -2214,10 +2222,11 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2227,19 +2236,20 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" +checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6" [[package]] name = "datafusion-execution" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -2249,16 +2259,19 @@ dependencies = [ "object_store", "parking_lot", "parquet", + "pin-project-lite", "rand 0.9.5", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e" dependencies = [ "arrow", "arrow-schema", @@ -2270,8 +2283,10 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -2279,21 +2294,21 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" +checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-ffi" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a4a09f24f1387408810acca070207a9cf6c7fc2368122982a8e7671f643d1e" +checksum = "ada11061028faad12bf1a45f2d5fa6624b7fcf65274fbf10a71ffe46c77cd10b" dependencies = [ "arrow", "arrow-schema", @@ -2324,13 +2339,13 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" +checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde" dependencies = [ "arrow", "arrow-buffer", - "base64 0.22.1", + "base64 0.23.0", "blake2", "blake3", "chrono", @@ -2343,7 +2358,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -2356,9 +2371,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b" dependencies = [ "arrow", "datafusion-common", @@ -2369,17 +2384,17 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", + "hashbrown 0.17.1", "log", "num-traits", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333" dependencies = [ "arrow", "datafusion-common", @@ -2389,9 +2404,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f" dependencies = [ "arrow", "arrow-ord", @@ -2406,7 +2421,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2414,9 +2429,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" +checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840" dependencies = [ "arrow", "async-trait", @@ -2430,9 +2445,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc" dependencies = [ "arrow", "datafusion-common", @@ -2447,9 +2462,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2457,20 +2472,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" dependencies = [ "datafusion-doc", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "datafusion-optimizer" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" +checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896" dependencies = [ "arrow", "chrono", @@ -2479,7 +2494,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "recursive", "regex", @@ -2488,9 +2503,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" +checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f" dependencies = [ "arrow", "datafusion-common", @@ -2498,10 +2513,11 @@ dependencies = [ "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -2510,9 +2526,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" +checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0" dependencies = [ "arrow", "datafusion-common", @@ -2520,31 +2536,32 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" +checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", + "datafusion-proto-models", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b" dependencies = [ "arrow", "datafusion-common", @@ -2555,15 +2572,16 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "datafusion-session", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" +checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112" dependencies = [ "arrow", "arrow-data", @@ -2571,6 +2589,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2580,26 +2599,28 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" +checksum = "0df0504eb9028d5e01f481af3519cde3f97ab650fd50ce7b787e94b69a7a193c" dependencies = [ "arrow", - "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", @@ -2615,26 +2636,38 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost", ] [[package]] name = "datafusion-proto-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" +checksum = "6b92415a2442964f180d39cdcd8ff1edd99f9260c1499ba80a396499c2154d11" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-proto-models" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e4c0bd6af4fcabdbe201ee86fbeef2ac423a44e1d8fda56d994c9e2d1d3ad2" +dependencies = [ + "datafusion-common", + "datafusion-proto-common", + "prost", +] + [[package]] name = "datafusion-pruning" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" +checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c" dependencies = [ "arrow", "datafusion-common", @@ -2648,10 +2681,11 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -2662,9 +2696,9 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" +checksum = "992dd0b954f24cb576cbeee3555c3083b9cf7a5a5f2448ea25f7a437d444163f" dependencies = [ "arrow", "bigdecimal", @@ -2692,9 +2726,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9" dependencies = [ "arrow", "bigdecimal", @@ -2707,13 +2741,14 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] name = "datafusion-sqllogictest" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d11faf3e95b017819a10800dc6d612c1f003696dea201700796b9268704f591" +checksum = "2c921345d249b784a513ad2be71fc3df18fd8f019fe6c82233f085a1871154fd" dependencies = [ "arrow", "async-trait", @@ -2724,7 +2759,7 @@ dependencies = [ "futures", "half", "indicatif", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "sqllogictest", @@ -2753,7 +2788,7 @@ dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2803,7 +2838,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2813,7 +2848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.119", ] [[package]] @@ -2875,7 +2910,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2941,7 +2976,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2982,7 +3017,7 @@ checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3281,7 +3316,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3404,7 +3439,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4022,7 +4057,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn", + "syn 2.0.119", "trybuild", ] @@ -4233,12 +4268,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "integer-encoding" version = "4.1.0" @@ -4288,6 +4317,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -4320,7 +4358,7 @@ checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4365,7 +4403,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.119", ] [[package]] @@ -4384,7 +4422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4654,6 +4692,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + [[package]] name = "marrow" version = "0.2.7" @@ -4804,7 +4851,7 @@ dependencies = [ "cfg-if 1.0.4", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4878,7 +4925,7 @@ checksum = "b40e46c845ac234bcba19db7ab252bc2778cbadd516a466d2f12b1580852d136" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4898,7 +4945,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4976,6 +5023,16 @@ dependencies = [ "serde", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -5055,7 +5112,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -5476,9 +5533,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.4.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" dependencies = [ "ahash", "arrow-array", @@ -5487,7 +5544,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.22.1", + "base64 0.23.0", "brotli", "bytes", "chrono", @@ -5495,17 +5552,15 @@ dependencies = [ "futures", "half", "hashbrown 0.17.1", - "lz4_flex", - "num-bigint", + "lz4_flex 0.14.0", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "ring", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -5593,7 +5648,7 @@ dependencies = [ "async-recursion", "bytes", "faststr", - "integer-encoding 4.1.0", + "integer-encoding", "lazy_static", "linkedbytes", "ordered-float 4.6.0", @@ -5621,7 +5676,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -5780,7 +5835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.119", ] [[package]] @@ -5818,10 +5873,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -5851,7 +5906,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -5869,9 +5924,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "fc153f5fd745cc038b5eed86622125969f8a39834a57bc96beaaf2512b1da729" dependencies = [ "libc", "once_cell", @@ -5883,18 +5938,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "bb77d9aa6d647507b55c69ee714d266d84c526c78ff0bc6dd8757f58591e64d1" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "3160087aa5733bce7d9a729f8c55f85a2a53b74742a09613178eeebff0722253" dependencies = [ "libc", "pyo3-build-config", @@ -5902,27 +5957,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "91f9d455db760a9a0b0ddeaac25f1390b8a36ba73dfbda9f127cac6fc340d4d5" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "e343bcec300ff262f5806a33a4e51b6d097a8a46435f512fcb83e95592581625" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -6152,7 +6206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -6210,7 +6264,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -6523,7 +6577,7 @@ checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -6582,7 +6636,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn", + "syn 2.0.119", "unicode-ident", ] @@ -6906,7 +6960,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -6915,6 +6969,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6930,7 +6985,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -6993,7 +7048,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7284,7 +7339,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7346,7 +7401,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.119", ] [[package]] @@ -7367,7 +7422,7 @@ dependencies = [ "sha2 0.10.9", "sqlx-core", "sqlx-sqlite", - "syn", + "syn 2.0.119", "tokio", "url", ] @@ -7505,7 +7560,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7578,7 +7633,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7590,7 +7645,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7626,6 +7681,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -7643,7 +7709,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7753,7 +7819,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7764,7 +7830,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7776,17 +7842,6 @@ dependencies = [ "cfg-if 1.0.4", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding 3.0.4", - "ordered-float 2.10.1", -] - [[package]] name = "time" version = "0.3.53" @@ -7876,7 +7931,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8095,7 +8150,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8197,7 +8252,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8233,7 +8288,7 @@ checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8531,7 +8586,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -8724,7 +8779,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8735,7 +8790,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -9038,7 +9093,7 @@ dependencies = [ "heapify", "itertools 0.14.0", "lazy_static", - "lz4_flex", + "lz4_flex 0.13.1", "more-asserts", "rand 0.10.2", "regex", @@ -9158,7 +9213,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -9179,7 +9234,7 @@ checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -9199,7 +9254,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -9239,7 +9294,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 31643aeb0a..8c220f489c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,15 +50,15 @@ aes-gcm = "0.10" anyhow = "1.0.72" apache-avro = { version = "0.21", features = ["snappy", "zstandard"] } array-init = "2" -arrow = "58.4" -arrow-arith = "58.4" -arrow-array = "58.4" -arrow-buffer = "58.4" -arrow-cast = "58.4" -arrow-ord = "58.4" -arrow-schema = "58.4" -arrow-select = "58.4" -arrow-string = "58.4" +arrow = "59.2" +arrow-arith = "59.2" +arrow-array = "59.2" +arrow-buffer = "59.2" +arrow-cast = "59.2" +arrow-ord = "59.2" +arrow-schema = "59.2" +arrow-select = "59.2" +arrow-string = "59.2" as-any = "0.3.2" async-trait = "0.1.89" aws-config = "1.8.7" @@ -79,10 +79,10 @@ chrono = "0.4.41" clap = { version = "4.5.48", features = ["derive", "cargo"] } crc32fast = "1" dashmap = "6" -datafusion = "54" -datafusion-cli = "54" -datafusion-ffi = "54" -datafusion-sqllogictest = "54" +datafusion = "55.0.0" +datafusion-cli = "55.0.0" +datafusion-ffi = "55.0.0" +datafusion-sqllogictest = "55.0.0" derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0" @@ -125,11 +125,11 @@ murmur3 = "0.5.2" once_cell = "1.20" opendal = "0.58" ordered-float = "4" -parquet = "58.4" +parquet = "59.2" pilota = "0.11.10" pretty_assertions = "1.4" proc-macro2 = "1" -pyo3 = "0.28" +pyo3 = "0.29" quote = "1" rand = "0.9.3" regex = "1.11.3" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 0f63c850b4..9681ed8dd9 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -75,7 +75,7 @@ filterwarnings = [ dev = [ "maturin>=1.9.4,<2.0", "pytest>=8.3.2", - "datafusion>=52,<55", + "datafusion==54.*", # Set a PyIceberg lower bound; otherwise the resolver may select the # unrelated 0.0.2 stub. # Keep pyarrow direct because current pyiceberg[pyarrow] releases depend on diff --git a/bindings/python/src/datafusion_table_provider.rs b/bindings/python/src/datafusion_table_provider.rs index 2f4745fa7a..153db63815 100644 --- a/bindings/python/src/datafusion_table_provider.rs +++ b/bindings/python/src/datafusion_table_provider.rs @@ -16,7 +16,7 @@ // under the License. use std::collections::HashMap; -use std::ffi::{CStr, CString}; +use std::ffi::CStr; use std::sync::Arc; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; @@ -32,7 +32,7 @@ use pyo3::types::{PyAny, PyCapsule}; use crate::runtime::runtime; -// pyo3 0.28's CapsuleName only exposes `unsafe fn as_cstr() -> &CStr`, +// CapsuleName only exposes `unsafe fn as_cstr() -> &CStr`, // so we accept &CStr to allow direct comparison without UTF-8 validation. pub(crate) fn validate_pycapsule(capsule: &Bound, name: &CStr) -> PyResult<()> { let capsule_name = capsule.name()?; @@ -121,7 +121,7 @@ impl PyIcebergDataFusionTable { py: Python<'py>, session: Bound<'py, PyAny>, ) -> PyResult> { - let capsule_name = CString::new("datafusion_table_provider").unwrap(); + let capsule_name = c"datafusion_table_provider"; let logical_codec = ffi_logical_codec_from_pycapsule(session)?; @@ -132,7 +132,7 @@ impl PyIcebergDataFusionTable { logical_codec, ); - PyCapsule::new(py, ffi_provider, Some(capsule_name)) + PyCapsule::new_with_value(py, ffi_provider, capsule_name) } } diff --git a/bindings/python/tests/test_datafusion_table_provider.py b/bindings/python/tests/test_datafusion_table_provider.py index d0a5d7cee3..fafef00f10 100644 --- a/bindings/python/tests/test_datafusion_table_provider.py +++ b/bindings/python/tests/test_datafusion_table_provider.py @@ -28,9 +28,9 @@ from pyiceberg.catalog import Catalog, load_catalog from pyiceberg_core.datafusion import IcebergDataFusionTable -if Version(datafusion.__version__) < Version("53.0.0"): +if Version(datafusion.__version__) < Version("55.0.0"): pytest.skip( - "Iceberg table provider requires datafusion>=53 for FFI compatibility", + "Iceberg table provider requires datafusion>=55 for FFI compatibility", allow_module_level=True, ) diff --git a/bindings/python/tests/test_huggingface_and_cdc.py b/bindings/python/tests/test_huggingface_and_cdc.py index 4bf5885654..226e6d0fd1 100644 --- a/bindings/python/tests/test_huggingface_and_cdc.py +++ b/bindings/python/tests/test_huggingface_and_cdc.py @@ -34,9 +34,10 @@ from pyiceberg.catalog import load_catalog from pyiceberg_core.datafusion import IcebergDataFusionTable -requires_datafusion_53 = pytest.mark.skipif( - Version(datafusion.__version__) < Version("53.0.0"), - reason="IcebergDataFusionTable requires datafusion>=53 for FFI compatibility", +requires_datafusion_55 = pytest.mark.skipif( + Version(datafusion.__version__) < Version("55.0.0"), + reason="IcebergDataFusionTable requires datafusion>=55 for FFI compatibility " + "with datafusion-ffi 55.0.0", ) @@ -107,7 +108,7 @@ def test_cdc_write_via_pyiceberg(local_catalog, sample_table): assert len(result) == len(sample_table) -@requires_datafusion_53 +@requires_datafusion_55 def test_cdc_write_and_read_via_datafusion(local_catalog, sample_table): """A table with CDC properties can be written and read back via DataFusion.""" local_catalog.create_namespace_if_not_exists("cdc_ns") @@ -182,7 +183,7 @@ def test_hf_cdc_write_and_read_via_pyarrow(hf_cdc_table, sample_table): @requires_hf -@requires_datafusion_53 +@requires_datafusion_55 def test_hf_cdc_write_and_read_via_datafusion(hf_cdc_table, sample_table): """PyIceberg writes CDC parquet to HF Hub; IcebergDataFusionTable reads it back via opendal-hf.""" tbl, token = hf_cdc_table diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock index 4676a7f63c..8e656ce507 100644 --- a/bindings/python/uv.lock +++ b/bindings/python/uv.lock @@ -881,7 +881,7 @@ dev = [ [package.metadata.requires-dev] dev = [ - { name = "datafusion", specifier = ">=52,<55" }, + { name = "datafusion", specifier = "==54.*" }, { name = "fastavro", specifier = ">=1.11.1" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "maturin", specifier = ">=1.9.4,<2.0" }, diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 56e11b1804..18256d362b 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -94,7 +94,7 @@ mockall = { workspace = true } pretty_assertions = { workspace = true } rand = { workspace = true } regex = { workspace = true } -serde_arrow = { version = "0.14", features = ["arrow-58"] } +serde_arrow = { version = "0.14", features = ["arrow-59"] } tempfile = { workspace = true } [package.metadata.cargo-machete] diff --git a/crates/integrations/datafusion/public-api.txt b/crates/integrations/datafusion/public-api.txt index e197c2057d..5fcb4e85d3 100644 --- a/crates/integrations/datafusion/public-api.txt +++ b/crates/integrations/datafusion/public-api.txt @@ -7,7 +7,7 @@ impl core::clone::Clone for iceberg_datafusion::metadata_table::IcebergMetadataT pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::clone(&self) -> iceberg_datafusion::metadata_table::IcebergMetadataTableProvider impl core::fmt::Debug for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider +impl datafusion_session::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _projection: core::option::Option<&'life2 alloc::vec::Vec>, _filters: &'life3 [datafusion_expr::expr::Expr], _limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType @@ -24,6 +24,7 @@ pub fn iceberg_datafusion::physical_plan::IcebergTableScan::fmt(&self, f: &mut c impl datafusion_physical_plan::display::DisplayAs for iceberg_datafusion::physical_plan::IcebergTableScan pub fn iceberg_datafusion::physical_plan::IcebergTableScan::fmt_as(&self, _t: datafusion_physical_plan::display::DisplayFormatType, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl datafusion_physical_plan::execution_plan::ExecutionPlan for iceberg_datafusion::physical_plan::IcebergTableScan +pub fn iceberg_datafusion::physical_plan::IcebergTableScan::apply_expressions(&self, _f: &mut dyn core::ops::function::FnMut(&alloc::sync::Arc) -> datafusion_common::error::Result) -> datafusion_common::error::Result pub fn iceberg_datafusion::physical_plan::IcebergTableScan::children(&self) -> alloc::vec::Vec<&alloc::sync::Arc<(dyn datafusion_physical_plan::execution_plan::ExecutionPlan + 'static)>> pub fn iceberg_datafusion::physical_plan::IcebergTableScan::execute(&self, _partition: usize, _context: alloc::sync::Arc) -> datafusion_common::error::Result pub fn iceberg_datafusion::physical_plan::IcebergTableScan::name(&self) -> &str @@ -40,7 +41,7 @@ impl core::clone::Clone for iceberg_datafusion::metadata_table::IcebergMetadataT pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::clone(&self) -> iceberg_datafusion::metadata_table::IcebergMetadataTableProvider impl core::fmt::Debug for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider +impl datafusion_session::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _projection: core::option::Option<&'life2 alloc::vec::Vec>, _filters: &'life3 [datafusion_expr::expr::Expr], _limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType @@ -53,8 +54,8 @@ impl core::default::Default for iceberg_datafusion::table_provider_factory::Iceb pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory +pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub struct iceberg_datafusion::table::IcebergStaticTableProvider impl iceberg_datafusion::IcebergStaticTableProvider pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table(table: iceberg::table::Table) -> iceberg::error::Result @@ -63,7 +64,7 @@ impl core::clone::Clone for iceberg_datafusion::IcebergStaticTableProvider pub fn iceberg_datafusion::IcebergStaticTableProvider::clone(&self) -> iceberg_datafusion::IcebergStaticTableProvider impl core::fmt::Debug for iceberg_datafusion::IcebergStaticTableProvider pub fn iceberg_datafusion::IcebergStaticTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider +impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider pub fn iceberg_datafusion::IcebergStaticTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_datafusion::IcebergStaticTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait pub fn iceberg_datafusion::IcebergStaticTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef @@ -74,7 +75,7 @@ impl core::clone::Clone for iceberg_datafusion::IcebergTableProvider pub fn iceberg_datafusion::IcebergTableProvider::clone(&self) -> iceberg_datafusion::IcebergTableProvider impl core::fmt::Debug for iceberg_datafusion::IcebergTableProvider pub fn iceberg_datafusion::IcebergTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProvider for iceberg_datafusion::IcebergTableProvider +impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergTableProvider pub fn iceberg_datafusion::IcebergTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, state: &'life1 dyn datafusion_session::session::Session, input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_datafusion::IcebergTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait pub fn iceberg_datafusion::IcebergTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef @@ -89,15 +90,15 @@ impl core::default::Default for iceberg_datafusion::table_provider_factory::Iceb pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory +pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub struct iceberg_datafusion::IcebergCatalogProvider impl iceberg_datafusion::IcebergCatalogProvider pub async fn iceberg_datafusion::IcebergCatalogProvider::try_new(client: alloc::sync::Arc) -> iceberg::error::Result impl core::fmt::Debug for iceberg_datafusion::IcebergCatalogProvider pub fn iceberg_datafusion::IcebergCatalogProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::catalog::CatalogProvider for iceberg_datafusion::IcebergCatalogProvider -pub fn iceberg_datafusion::IcebergCatalogProvider::schema(&self, name: &str) -> core::option::Option> +impl datafusion_session::catalog::CatalogProvider for iceberg_datafusion::IcebergCatalogProvider +pub fn iceberg_datafusion::IcebergCatalogProvider::schema(&self, name: &str) -> core::option::Option> pub fn iceberg_datafusion::IcebergCatalogProvider::schema_names(&self) -> alloc::vec::Vec pub struct iceberg_datafusion::IcebergStaticTableProvider impl iceberg_datafusion::IcebergStaticTableProvider @@ -107,7 +108,7 @@ impl core::clone::Clone for iceberg_datafusion::IcebergStaticTableProvider pub fn iceberg_datafusion::IcebergStaticTableProvider::clone(&self) -> iceberg_datafusion::IcebergStaticTableProvider impl core::fmt::Debug for iceberg_datafusion::IcebergStaticTableProvider pub fn iceberg_datafusion::IcebergStaticTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider +impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider pub fn iceberg_datafusion::IcebergStaticTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_datafusion::IcebergStaticTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait pub fn iceberg_datafusion::IcebergStaticTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef @@ -118,7 +119,7 @@ impl core::clone::Clone for iceberg_datafusion::IcebergTableProvider pub fn iceberg_datafusion::IcebergTableProvider::clone(&self) -> iceberg_datafusion::IcebergTableProvider impl core::fmt::Debug for iceberg_datafusion::IcebergTableProvider pub fn iceberg_datafusion::IcebergTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProvider for iceberg_datafusion::IcebergTableProvider +impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergTableProvider pub fn iceberg_datafusion::IcebergTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, state: &'life1 dyn datafusion_session::session::Session, input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_datafusion::IcebergTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait pub fn iceberg_datafusion::IcebergTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef @@ -132,7 +133,7 @@ impl core::default::Default for iceberg_datafusion::table_provider_factory::Iceb pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_catalog::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory +pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_datafusion::from_datafusion_error(error: datafusion_common::error::DataFusionError) -> iceberg::error::Error pub fn iceberg_datafusion::to_datafusion_error(error: iceberg::error::Error) -> datafusion_common::error::DataFusionError diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs index f1d93679e0..bb507217e2 100644 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ b/crates/integrations/datafusion/src/physical_plan/commit.rs @@ -22,9 +22,10 @@ use datafusion::arrow::array::{Array, ArrayRef, RecordBatch, StringArray, UInt64 use datafusion::arrow::datatypes::{ DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, }; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{DataFusionError, Result as DFResult}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; +use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; @@ -136,6 +137,13 @@ impl ExecutionPlan for IcebergCommitExec { vec![&self.input] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn required_input_distribution(&self) -> Vec { vec![datafusion::physical_plan::Distribution::SinglePartition; self.children().len()] } @@ -338,6 +346,13 @@ mod tests { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, diff --git a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs b/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs index f15acf807b..33c1143ab3 100644 --- a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs @@ -18,7 +18,8 @@ use std::sync::Arc; use datafusion::catalog::TableProvider; -use datafusion::physical_expr::EquivalenceProperties; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; @@ -70,6 +71,13 @@ impl ExecutionPlan for IcebergMetadataScan { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion::error::Result, + ) -> datafusion::error::Result { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index 1e53c20cec..1796304552 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -21,9 +21,10 @@ use std::vec; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::error::Result as DFResult; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; @@ -125,6 +126,13 @@ impl ExecutionPlan for IcebergTableScan { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 3d2ea32226..40ad9fad00 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -24,9 +24,10 @@ use datafusion::arrow::datatypes::{ DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, }; use datafusion::common::Result as DFResult; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::error::DataFusionError; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; +use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ @@ -155,6 +156,13 @@ impl ExecutionPlan for IcebergWriteExec { vec![&self.input] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, @@ -389,6 +397,13 @@ mod tests { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, diff --git a/crates/integrations/datafusion/src/table/table_provider_factory.rs b/crates/integrations/datafusion/src/table/table_provider_factory.rs index 9692340cac..a3b293f33a 100644 --- a/crates/integrations/datafusion/src/table/table_provider_factory.rs +++ b/crates/integrations/datafusion/src/table/table_provider_factory.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::catalog::{Session, TableProvider, TableProviderFactory}; +use datafusion::common::TableReference; use datafusion::error::Result as DFResult; use datafusion::logical_expr::CreateExternalTable; -use datafusion::sql::TableReference; use iceberg::io::{FileIOBuilder, LocalFsStorageFactory, StorageFactory}; use iceberg::table::StaticTable; use iceberg::{Error, ErrorKind, Result, TableIdent}; @@ -41,9 +41,9 @@ use crate::to_datafusion_error; /// ``` /// use std::sync::Arc; /// +/// use datafusion::common::TableReference; /// use datafusion::execution::session_state::SessionStateBuilder; /// use datafusion::prelude::*; -/// use datafusion::sql::TableReference; /// use iceberg_datafusion::IcebergTableProviderFactory; /// /// #[tokio::main] @@ -123,10 +123,9 @@ impl TableProviderFactory for IcebergTableProviderFactory { _state: &dyn Session, cmd: &CreateExternalTable, ) -> DFResult> { - check_cmd(cmd).map_err(to_datafusion_error)?; + let metadata_file_path = check_cmd(cmd).map_err(to_datafusion_error)?; let table_name = &cmd.name; - let metadata_file_path = &cmd.location; let options = &cmd.options; let table_name_with_ns = complement_namespace_if_necessary(table_name); @@ -154,7 +153,7 @@ impl TableProviderFactory for IcebergTableProviderFactory { } } -fn check_cmd(cmd: &CreateExternalTable) -> Result<()> { +fn check_cmd(cmd: &CreateExternalTable) -> Result<&str> { let CreateExternalTable { schema, table_partition_cols, @@ -178,7 +177,13 @@ fn check_cmd(cmd: &CreateExternalTable) -> Result<()> { )); } - Ok(()) + match cmd.locations.as_slice() { + [location] => Ok(location), + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Iceberg external tables require exactly one metadata location.", + )), + } } /// Complements the namespace of a table name if necessary. @@ -218,12 +223,11 @@ mod tests { use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::catalog::TableProviderFactory; - use datafusion::common::{Constraints, DFSchema}; + use datafusion::common::{Constraints, DFSchema, TableReference}; use datafusion::execution::session_state::SessionStateBuilder; use datafusion::logical_expr::CreateExternalTable; use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; use datafusion::prelude::SessionContext; - use datafusion::sql::TableReference; use super::*; @@ -257,7 +261,7 @@ mod tests { CreateExternalTable { name: TableReference::partial("static_ns", "static_table"), - location: metadata_file_path, + locations: vec![metadata_file_path], schema: Arc::new(DFSchema::empty()), file_type: "iceberg".to_string(), options: Default::default(), From 75c9201798ae69a9c1c0048df2e1f06e5271a907 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 3 Sep 2026 16:47:46 -0700 Subject: [PATCH 27/74] refactor(pyiceberg-core): remove DataFusion TableProvider export (#3143) * refactor(python): remove DataFusion integration Remove the DataFusion table provider export, its tests, and its Rust and Python dependency trees from pyiceberg-core. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * chore(python): leave dependency inventory unchanged Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs(python): clarify PyArrow test dependency Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * docs(python): explain PyIceberg core conflict Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 176 --------- bindings/python/Cargo.toml | 6 +- bindings/python/pyproject.toml | 14 +- .../python/src/datafusion_table_provider.rs | 150 -------- bindings/python/src/lib.rs | 3 - bindings/python/src/runtime.rs | 32 -- .../tests/test_datafusion_table_provider.py | 163 --------- .../python/tests/test_huggingface_and_cdc.py | 197 ---------- bindings/python/uv.lock | 339 +----------------- 9 files changed, 7 insertions(+), 1073 deletions(-) delete mode 100644 bindings/python/src/datafusion_table_provider.rs delete mode 100644 bindings/python/src/runtime.rs delete mode 100644 bindings/python/tests/test_datafusion_table_provider.py delete mode 100644 bindings/python/tests/test_huggingface_and_cdc.py diff --git a/Cargo.lock b/Cargo.lock index 84a87465f8..99557339a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -528,12 +528,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-ffi" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" - [[package]] name = "async-lock" version = "3.4.2" @@ -2092,7 +2086,6 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", - "datafusion-proto-models", "datafusion-session", "flate2", "futures", @@ -2126,7 +2119,6 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "datafusion-proto-models", "datafusion-session", "futures", "itertools 0.15.0", @@ -2169,7 +2161,6 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -2193,7 +2184,6 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -2222,7 +2212,6 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", - "datafusion-proto-models", "datafusion-pruning", "datafusion-session", "futures", @@ -2283,8 +2272,6 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "datafusion-proto-common", - "datafusion-proto-models", "indexmap 2.14.0", "itertools 0.15.0", "recursive", @@ -2304,39 +2291,6 @@ dependencies = [ "itertools 0.15.0", ] -[[package]] -name = "datafusion-ffi" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ada11061028faad12bf1a45f2d5fa6624b7fcf65274fbf10a71ffe46c77cd10b" -dependencies = [ - "arrow", - "arrow-schema", - "async-ffi", - "async-trait", - "chrono", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-proto", - "datafusion-proto-common", - "datafusion-session", - "futures", - "libloading", - "log", - "prost", - "semver", - "stabby", - "tokio", -] - [[package]] name = "datafusion-functions" version = "55.0.0" @@ -2513,7 +2467,6 @@ dependencies = [ "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", - "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap 2.14.0", @@ -2549,7 +2502,6 @@ dependencies = [ "chrono", "datafusion-common", "datafusion-expr-common", - "datafusion-proto-models", "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.15.0", @@ -2599,8 +2551,6 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", - "datafusion-proto-common", - "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", @@ -2614,55 +2564,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "datafusion-proto" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0df0504eb9028d5e01f481af3519cde3f97ab650fd50ce7b787e94b69a7a193c" -dependencies = [ - "arrow", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-datasource", - "datafusion-datasource-arrow", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-table", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-proto-common", - "datafusion-proto-models", - "object_store", - "prost", -] - -[[package]] -name = "datafusion-proto-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92415a2442964f180d39cdcd8ff1edd99f9260c1499ba80a396499c2154d11" -dependencies = [ - "arrow", - "datafusion-common", - "prost", -] - -[[package]] -name = "datafusion-proto-models" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e4c0bd6af4fcabdbe201ee86fbeef2ac423a44e1d8fda56d994c9e2d1d3ad2" -dependencies = [ - "datafusion-common", - "datafusion-proto-common", - "prost", -] - [[package]] name = "datafusion-pruning" version = "55.0.0" @@ -4541,16 +4442,6 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "libloading" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" -dependencies = [ - "cfg-if 1.0.4", - "windows-link", -] - [[package]] name = "liblzma" version = "0.4.7" @@ -5856,29 +5747,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" -dependencies = [ - "anyhow", - "itertools 0.13.0", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "psm" version = "0.1.31" @@ -5914,12 +5782,8 @@ name = "pyiceberg_core_rust" version = "0.10.1" dependencies = [ "arrow", - "datafusion-ffi", "iceberg", - "iceberg-datafusion", - "iceberg-storage-opendal", "pyo3", - "tokio", ] [[package]] @@ -7105,12 +6969,6 @@ dependencies = [ "cc", ] -[[package]] -name = "sha2-const-stable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" - [[package]] name = "sharded-slab" version = "0.1.7" @@ -7529,40 +7387,6 @@ dependencies = [ "url", ] -[[package]] -name = "stabby" -version = "72.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec" -dependencies = [ - "rustversion", - "stabby-abi", -] - -[[package]] -name = "stabby-abi" -version = "72.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4" -dependencies = [ - "rustc_version", - "rustversion", - "sha2-const-stable", - "stabby-macros", -] - -[[package]] -name = "stabby-macros" -version = "72.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 38bb310c4f..96e1f1ced8 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -34,13 +34,9 @@ repository = { workspace = true } crate-type = ["cdylib"] [dependencies] -arrow = { workspace = true, features = ["pyarrow"] } -datafusion-ffi = { workspace = true } +arrow = { workspace = true, features = ["chrono-tz", "pyarrow"] } iceberg = { workspace = true } -iceberg-datafusion = { workspace = true } -iceberg-storage-opendal = { workspace = true, features = ["opendal-all"] } pyo3 = { workspace = true, features = ["abi3-py310"] } -tokio = { workspace = true } [package.metadata.cargo-machete] ignored = [] diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 9681ed8dd9..017ffaf040 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -65,23 +65,17 @@ config = [ ignore = ["F403", "F405"] [tool.pytest.ini_options] -filterwarnings = [ - "error", - # huggingface_hub uses hf_xet.upload_files() internally which is deprecated in hf_xet - "ignore::DeprecationWarning:huggingface_hub", -] +filterwarnings = ["error"] [dependency-groups] dev = [ "maturin>=1.9.4,<2.0", "pytest>=8.3.2", - "datafusion==54.*", # Set a PyIceberg lower bound; otherwise the resolver may select the # unrelated 0.0.2 stub. - # Keep pyarrow direct because current pyiceberg[pyarrow] releases depend on - # pyiceberg-core, which conflicts with this local pyiceberg-core project. - "pyiceberg[sql-sqlite]>=0.11", + # Install PyArrow directly instead of using pyiceberg[pyarrow]: that extra + # depends on the published pyiceberg-core, which conflicts with this local build. + "pyiceberg>=0.11", "pyarrow>=17", "fastavro>=1.11.1", - "huggingface_hub>=0.20", ] diff --git a/bindings/python/src/datafusion_table_provider.rs b/bindings/python/src/datafusion_table_provider.rs deleted file mode 100644 index 153db63815..0000000000 --- a/bindings/python/src/datafusion_table_provider.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::ffi::CStr; -use std::sync::Arc; - -use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; -use datafusion_ffi::table_provider::FFI_TableProvider; -use iceberg::TableIdent; -use iceberg::io::FileIOBuilder; -use iceberg::table::StaticTable; -use iceberg_datafusion::table::IcebergStaticTableProvider; -use iceberg_storage_opendal::OpenDalResolvingStorageFactory; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::*; -use pyo3::types::{PyAny, PyCapsule}; - -use crate::runtime::runtime; - -// CapsuleName only exposes `unsafe fn as_cstr() -> &CStr`, -// so we accept &CStr to allow direct comparison without UTF-8 validation. -pub(crate) fn validate_pycapsule(capsule: &Bound, name: &CStr) -> PyResult<()> { - let capsule_name = capsule.name()?; - match capsule_name { - None => Err(PyValueError::new_err( - "Expected PyCapsule to have name set.", - )), - Some(capsule_name) if unsafe { capsule_name.as_cstr() } != name => { - Err(PyValueError::new_err("PyCapsule name mismatch")) - } - _ => Ok(()), - } -} - -pub(crate) fn ffi_logical_codec_from_pycapsule( - obj: Bound, -) -> PyResult { - let attr_name = "__datafusion_logical_extension_codec__"; - let capsule = if obj.hasattr(attr_name)? { - obj.getattr(attr_name)?.call0()? - } else { - obj - }; - - let capsule_name = c"datafusion_logical_extension_codec"; - let capsule = capsule.cast::()?; - validate_pycapsule(capsule, capsule_name)?; - - let ptr = capsule.pointer_checked(Some(capsule_name))?; - let codec = unsafe { &*(ptr.as_ptr() as *const FFI_LogicalExtensionCodec) }; - Ok(codec.clone()) -} - -#[pyclass(name = "IcebergDataFusionTable")] -pub struct PyIcebergDataFusionTable { - inner: Arc, -} - -#[pymethods] -impl PyIcebergDataFusionTable { - #[new] - fn new( - identifier: Vec, - metadata_location: String, - file_io_properties: Option>, - ) -> PyResult { - let runtime = runtime(); - - let provider = runtime.block_on(async { - let table_ident = TableIdent::from_strs(identifier) - .map_err(|e| PyRuntimeError::new_err(format!("Invalid table identifier: {e}")))?; - - let factory = Arc::new(OpenDalResolvingStorageFactory::new()); - - let mut builder = FileIOBuilder::new(factory); - - if let Some(props) = file_io_properties { - builder = builder.with_props(props); - } - - let file_io = builder.build(); - - let static_table = - StaticTable::from_metadata_file(&metadata_location, table_ident, file_io) - .await - .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to load static table: {e}")) - })?; - - let table = static_table.into_table(); - - IcebergStaticTableProvider::try_new_from_table(table) - .await - .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to create table provider: {e}")) - }) - })?; - - Ok(Self { - inner: Arc::new(provider), - }) - } - - fn __datafusion_table_provider__<'py>( - &self, - py: Python<'py>, - session: Bound<'py, PyAny>, - ) -> PyResult> { - let capsule_name = c"datafusion_table_provider"; - - let logical_codec = ffi_logical_codec_from_pycapsule(session)?; - - let ffi_provider = FFI_TableProvider::new_with_ffi_codec( - self.inner.clone(), - false, - Some(runtime()), - logical_codec, - ); - - PyCapsule::new_with_value(py, ffi_provider, capsule_name) - } -} - -pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let this = PyModule::new(py, "datafusion")?; - - this.add_class::()?; - - m.add_submodule(&this)?; - py.import("sys")? - .getattr("modules")? - .set_item("pyiceberg_core.datafusion", this)?; - - Ok(()) -} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 131d864bdd..baddbe93a0 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -18,15 +18,12 @@ use pyo3::prelude::*; mod data_file; -mod datafusion_table_provider; mod error; mod manifest; -mod runtime; mod transform; #[pymodule] fn pyiceberg_core_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - datafusion_table_provider::register_module(py, m)?; transform::register_module(py, m)?; manifest::register_module(py, m)?; Ok(()) diff --git a/bindings/python/src/runtime.rs b/bindings/python/src/runtime.rs deleted file mode 100644 index 81d5b6dd6f..0000000000 --- a/bindings/python/src/runtime.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::OnceLock; - -use tokio::runtime::{Handle, Runtime}; - -static RUNTIME: OnceLock = OnceLock::new(); - -pub fn runtime() -> Handle { - match Handle::try_current() { - Ok(h) => h.clone(), - _ => { - let rt = RUNTIME.get_or_init(|| Runtime::new().unwrap()); - rt.handle().clone() - } - } -} diff --git a/bindings/python/tests/test_datafusion_table_provider.py b/bindings/python/tests/test_datafusion_table_provider.py deleted file mode 100644 index fafef00f10..0000000000 --- a/bindings/python/tests/test_datafusion_table_provider.py +++ /dev/null @@ -1,163 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import uuid -from datetime import date, datetime -from pathlib import Path - -import datafusion -import pyarrow as pa -import pytest -from datafusion import SessionContext -from packaging.version import Version -from pyiceberg.catalog import Catalog, load_catalog -from pyiceberg_core.datafusion import IcebergDataFusionTable - -if Version(datafusion.__version__) < Version("55.0.0"): - pytest.skip( - "Iceberg table provider requires datafusion>=55 for FFI compatibility", - allow_module_level=True, - ) - - -@pytest.fixture(scope="session") -def warehouse(tmp_path_factory: pytest.TempPathFactory) -> Path: - return tmp_path_factory.mktemp("warehouse") - - -@pytest.fixture(scope="session") -def catalog(warehouse: Path): - catalog = load_catalog( - "default", - uri=f"sqlite:///{warehouse}/pyiceberg_catalog.db", - warehouse=f"file://{warehouse}", - ) - yield catalog - catalog.close() - - -@pytest.fixture(scope="session") -def arrow_table_with_null() -> "pa.Table": - """Pyarrow table with all kinds of columns.""" - import pyarrow as pa - - return pa.Table.from_pydict( - { - "bool": [False, None, True], - "string": ["a", None, "z"], - # Go over the 16 bytes to kick in truncation - "string_long": ["a" * 22, None, "z" * 22], - "int": [1, None, 9], - "long": [1, None, 9], - "float": [0.0, None, 0.9], - "double": [0.0, None, 0.9], - # 'time': [1_000_000, None, 3_000_000], # Example times: 1s, none, and 3s past midnight #Spark does not support time fields - "timestamp": [ - datetime(2023, 1, 1, 19, 25, 00), # noqa: DTZ001 - None, - datetime(2023, 3, 1, 19, 25, 00), # noqa: DTZ001 - ], - # "timestamptz": [ - # datetime(2023, 1, 1, 19, 25, 00, tzinfo=timezone.utc), - # None, - # datetime(2023, 3, 1, 19, 25, 00, tzinfo=timezone.utc), - # ], - "date": [date(2023, 1, 1), None, date(2023, 3, 1)], - # Not supported by Spark - # 'time': [time(1, 22, 0), None, time(19, 25, 0)], - # Not natively supported by Arrow - # 'uuid': [uuid.UUID('00000000-0000-0000-0000-000000000000').bytes, None, uuid.UUID('11111111-1111-1111-1111-111111111111').bytes], - "binary": [b"\01", None, b"\22"], - "fixed": [ - uuid.UUID("00000000-0000-0000-0000-000000000000").bytes, - None, - uuid.UUID("11111111-1111-1111-1111-111111111111").bytes, - ], - }, - ) - - -def test_register_iceberg_table_provider( - catalog: Catalog, arrow_table_with_null: pa.Table -) -> None: - catalog.create_namespace_if_not_exists("default") - iceberg_table = catalog.create_table_if_not_exists( - "default.dataset", - schema=arrow_table_with_null.schema, - ) - iceberg_table.append(arrow_table_with_null) - - iceberg_table_provider = IcebergDataFusionTable( - identifier=iceberg_table.name(), - metadata_location=iceberg_table.metadata_location, - file_io_properties=iceberg_table.io.properties, - ) - - ctx = SessionContext() - ctx.register_table("test", iceberg_table_provider) - - datafusion_table = ctx.table("test") - assert datafusion_table is not None - - # check that the schema is the same - from pyiceberg.io.pyarrow import _pyarrow_schema_ensure_small_types - - assert _pyarrow_schema_ensure_small_types( - datafusion_table.schema() - ) == _pyarrow_schema_ensure_small_types(iceberg_table.schema().as_arrow()) - # large/small type mismatches, fixed in pyiceberg 0.10.0 - # assert datafusion_table.schema() == iceberg_table.schema().as_arrow() - - # check that the data is the same - assert ( - datafusion_table.to_arrow_table().to_pylist() - == iceberg_table.scan().to_arrow().to_pylist() - ) - # large/small type mismatches, fixed in pyiceberg 0.10.0 - # assert datafusion_table.to_arrow_table() == iceberg_table.scan().to_arrow() - - -def test_register_pyiceberg_table( - catalog: Catalog, arrow_table_with_null: pa.Table -) -> None: - from types import MethodType - - catalog.create_namespace_if_not_exists("default") - iceberg_table = catalog.create_table_if_not_exists( - "default.dataset", - schema=arrow_table_with_null.schema, - ) - iceberg_table.append(arrow_table_with_null) - - # monkey patch the __datafusion_table_provider__ method to the iceberg table - def __datafusion_table_provider__(self, session): - return IcebergDataFusionTable( - identifier=self.name(), - metadata_location=self.metadata_location, - file_io_properties=self.io.properties, - ).__datafusion_table_provider__(session) - - iceberg_table.__datafusion_table_provider__ = MethodType( - __datafusion_table_provider__, iceberg_table - ) - - ctx = SessionContext() - ctx.register_table("test", iceberg_table) - - datafusion_table = ctx.table("test") - assert datafusion_table is not None diff --git a/bindings/python/tests/test_huggingface_and_cdc.py b/bindings/python/tests/test_huggingface_and_cdc.py deleted file mode 100644 index 226e6d0fd1..0000000000 --- a/bindings/python/tests/test_huggingface_and_cdc.py +++ /dev/null @@ -1,197 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Tests for HuggingFace Hub URI support and CDC (content-defined chunking) options. - -CDC options are standard Iceberg table properties and work in both Rust and PyIceberg -automatically — no special API is required beyond setting string properties. - -HF credentials are passed as file_io_properties to IcebergDataFusionTable. -Tests requiring live HF credentials are skipped when HF_TOKEN or HF_DATASET is not set. -""" - -import os - -import datafusion -import pyarrow as pa -import pytest -from datafusion import SessionContext -from packaging.version import Version -from pyiceberg.catalog import load_catalog -from pyiceberg_core.datafusion import IcebergDataFusionTable - -requires_datafusion_55 = pytest.mark.skipif( - Version(datafusion.__version__) < Version("55.0.0"), - reason="IcebergDataFusionTable requires datafusion>=55 for FFI compatibility " - "with datafusion-ffi 55.0.0", -) - - -# --------------------------------------------------------------------------- -# CDC tests — run without any external credentials -# --------------------------------------------------------------------------- - - -@pytest.fixture(scope="module") -def local_catalog(tmp_path_factory: pytest.TempPathFactory): - warehouse = tmp_path_factory.mktemp("cdc_warehouse") - catalog = load_catalog( - "default", - uri=f"sqlite:///{warehouse}/pyiceberg_catalog.db", - warehouse=f"file://{warehouse}", - ) - yield catalog - catalog.close() - - -@pytest.fixture(scope="module") -def sample_table() -> pa.Table: - return pa.table( - { - "id": pa.array(list(range(1000)), type=pa.int32()), - "payload": pa.array( - [f"row-{i:06d}" for i in range(1000)], type=pa.large_utf8() - ), - } - ) - - -def test_cdc_table_properties_are_persisted(local_catalog, sample_table): - """Table properties with CDC options are stored and returned as-is.""" - local_catalog.create_namespace_if_not_exists("cdc_ns") - - # Use values that differ from parquet defaults (256 KiB min, 1 MiB max, 0 norm). - tbl = local_catalog.create_table_if_not_exists( - "cdc_ns.cdc_persist", - schema=sample_table.schema, - properties={ - "write.parquet.content-defined-chunking.min-chunk-size": "65536", - "write.parquet.content-defined-chunking.max-chunk-size": "524288", - "write.parquet.content-defined-chunking.norm-level": "2", - }, - ) - - props = tbl.properties - assert props.get("write.parquet.content-defined-chunking.min-chunk-size") == "65536" - assert ( - props.get("write.parquet.content-defined-chunking.max-chunk-size") == "524288" - ) - assert props.get("write.parquet.content-defined-chunking.norm-level") == "2" - - -def test_cdc_write_via_pyiceberg(local_catalog, sample_table): - """PyIceberg tbl.append() writes parquet with CDC options when properties are set.""" - local_catalog.create_namespace_if_not_exists("cdc_ns") - - tbl = local_catalog.create_table_if_not_exists( - "cdc_ns.cdc_pyiceberg_write", - schema=sample_table.schema, - properties={"write.parquet.content-defined-chunking.enabled": "true"}, - ) - tbl.append(sample_table) - - result = tbl.scan().to_arrow() - assert len(result) == len(sample_table) - - -@requires_datafusion_55 -def test_cdc_write_and_read_via_datafusion(local_catalog, sample_table): - """A table with CDC properties can be written and read back via DataFusion.""" - local_catalog.create_namespace_if_not_exists("cdc_ns") - - tbl = local_catalog.create_table_if_not_exists( - "cdc_ns.cdc_write_read", - schema=sample_table.schema, - properties={"write.parquet.content-defined-chunking.enabled": "true"}, - ) - tbl.append(sample_table) - - provider = IcebergDataFusionTable( - identifier=tbl.name(), - metadata_location=tbl.metadata_location, - file_io_properties=tbl.io.properties, - ) - - ctx = SessionContext() - ctx.register_table("cdc_table", provider) - assert ctx.table("cdc_table").count() == len(sample_table) - - -# --------------------------------------------------------------------------- -# HF + CDC tests — skipped when HF_TOKEN or HF_DATASET is not set -# --------------------------------------------------------------------------- - -requires_hf = pytest.mark.skipif( - not os.environ.get("HF_TOKEN") or not os.environ.get("HF_DATASET"), - reason="HF_TOKEN or HF_DATASET not set", -) - - -@pytest.fixture(scope="module") -def hf_cdc_table(sample_table): - """Write a CDC-enabled Iceberg table to HF Hub once; share across HF tests. - - Uses FsspecFileIO backed by huggingface_hub's HfFileSystem (hf:// in fsspec). - HF_TOKEN is read from the environment automatically by HfFileSystem. - """ - token = os.environ["HF_TOKEN"] - dataset = os.environ["HF_DATASET"] - - warehouse = f"hf://datasets/{dataset}/iceberg-ci-{os.getpid()}" - catalog = load_catalog( - "hf_test", - **{ - "uri": "sqlite:///:memory:", - "warehouse": warehouse, - "py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", - }, - ) - catalog.create_namespace("ns") - tbl = catalog.create_table( - "ns.cdc_tbl", - schema=sample_table.schema, - properties={"write.parquet.content-defined-chunking.enabled": "true"}, - ) - tbl.append(sample_table) - # HfFileSystem.dircache may reflect the pre-write state; invalidate it so - # subsequent reads (info/open) see the files just uploaded via xet. - tbl.io.get_fs("hf").invalidate_cache() - yield tbl, token - catalog.close() - - -@requires_hf -def test_hf_cdc_write_and_read_via_pyarrow(hf_cdc_table, sample_table): - """PyIceberg writes CDC parquet to HF Hub; PyArrow scan reads it back.""" - tbl, _ = hf_cdc_table - result = tbl.scan().to_arrow() - assert len(result) == len(sample_table) - - -@requires_hf -@requires_datafusion_55 -def test_hf_cdc_write_and_read_via_datafusion(hf_cdc_table, sample_table): - """PyIceberg writes CDC parquet to HF Hub; IcebergDataFusionTable reads it back via opendal-hf.""" - tbl, token = hf_cdc_table - provider = IcebergDataFusionTable( - identifier=tbl.name(), - metadata_location=tbl.metadata_location, - file_io_properties={"hf.token": token}, - ) - ctx = SessionContext() - ctx.register_table("hf_table", provider) - assert ctx.table("hf_table").count() == len(sample_table) diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock index 8e656ce507..175404bad4 100644 --- a/bindings/python/uv.lock +++ b/bindings/python/uv.lock @@ -15,20 +15,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] -[[package]] -name = "anyio" -version = "4.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "idna" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, -] - [[package]] name = "cachetools" version = "6.2.6" @@ -164,15 +150,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -182,29 +159,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "datafusion" -version = "54.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cloudpickle" }, - { name = "pyarrow" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" }, - { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" }, - { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" }, - { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" }, - { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" }, - { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" }, - { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" }, - { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" }, - { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" }, - { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" }, -] - [[package]] name = "exceptiongroup" version = "1.3.1" @@ -269,15 +223,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/f1/f21bd5319113e89ceceed2df840df21e9c5150d181db74b6ba80400f9f48/fastavro-1.12.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:afede7324822800e4f90e96b9514188a237a60f35e8e7a10b2129c10c78f6e4d", size = 3356664, upload-time = "2026-04-24T14:37:34.231Z" }, ] -[[package]] -name = "filelock" -version = "3.29.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, -] - [[package]] name = "fsspec" version = "2026.4.0" @@ -287,141 +232,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] -[[package]] -name = "greenlet" -version = "3.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3c/3f/dbf99fb14bfeb88c28f16729215478c0e265cacd6dc22270c8f31bb6892f/greenlet-3.5.0.tar.gz", hash = "sha256:d419647372241bc68e957bf38d5c1f98852155e4146bd1e4121adea81f4f01e4", size = 196995, upload-time = "2026-04-27T13:37:15.544Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/03/84359833f7e1d49a883e92777637c592306030e30cee5e2b1e6476f95c88/greenlet-3.5.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:29ea813b2e1f45fa9649a17853b2b5465c4072fbcb072e5af6cd3a288216574a", size = 283502, upload-time = "2026-04-27T12:20:55.213Z" }, - { url = "https://files.pythonhosted.org/packages/25/ce/6f9f008266273aa14a2e011945797ac5802b97b8b40efe7afe1ee6c1afc9/greenlet-3.5.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:804a70b328e706b785c6ef16187051c394a63dd1a906d89be24b6ad77759f13f", size = 600508, upload-time = "2026-04-27T12:52:37.876Z" }, - { url = "https://files.pythonhosted.org/packages/e0/6d/b0f3272c2368ea2c1aa19a5ad70db0be8f8dff6e6d3d1eb82efa00cbcf19/greenlet-3.5.0-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:884f649de075b84739713d41dd4dfd41e2b910bfb769c4a3ea02ec1da52cd9bb", size = 613283, upload-time = "2026-04-27T12:59:37.957Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ac/0b509b6fb93551ce5a01612ee1acda7f7dda4bbb66c99aeb2ab403d205dc/greenlet-3.5.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b28037cb07768933c54d81bfe47a85f9f402f57d7d69743b991a713b63954eb", size = 613418, upload-time = "2026-04-27T12:25:23.852Z" }, - { url = "https://files.pythonhosted.org/packages/03/03/2b2b680ec87aaa97998fb5b8d76658d4d3560386864f17efab33ba7c2e24/greenlet-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cda05425526240807408156b6960a17a79a0c760b813573b67027823be760977", size = 1572229, upload-time = "2026-04-27T12:53:23.509Z" }, - { url = "https://files.pythonhosted.org/packages/61/e4/42b259e7a19aff1a270a4bd82caf6353109ed6860c9454e18f37162b83ae/greenlet-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9c615f869163e14bb1ced20322d8038fb680b08236521ac3f30cd4c1288785a0", size = 1639886, upload-time = "2026-04-27T12:25:22.325Z" }, - { url = "https://files.pythonhosted.org/packages/6f/b4/733ca47b883b67c57f90d3ecb21055c9ec753597d10754ac201644061f9d/greenlet-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:ba8f0bdc2fae6ce915dfd0c16d2d00bca7e4247c1eae4416e06430e522137858", size = 237795, upload-time = "2026-04-27T12:21:40.118Z" }, - { url = "https://files.pythonhosted.org/packages/8b/0f/a91f143f356523ff682309732b175765a9bc2836fd7c081c2c67fedc1ad4/greenlet-3.5.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8f1cc966c126639cd152fdaa52624d2655f492faa79e013fea161de3e6dda082", size = 284726, upload-time = "2026-04-27T12:20:51.402Z" }, - { url = "https://files.pythonhosted.org/packages/95/82/800646c7ffc5dbabd75ddd2f6b519bb898c0c9c969e5d0473bfe5d20bcce/greenlet-3.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:362624e6a8e5bca3b8233e45eef33903a100e9539a2b995c364d595dbc4018b3", size = 604264, upload-time = "2026-04-27T12:52:39.494Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/354867c0bba812fc33b15bc55aedafedd0aee3c7dd91dfca22444157dc0c/greenlet-3.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5ecd83806b0f4c2f53b1018e0005cd82269ea01d42befc0368730028d850ed1c", size = 616099, upload-time = "2026-04-27T12:59:39.623Z" }, - { url = "https://files.pythonhosted.org/packages/ff/b0/815bece7399e01cadb69014219eebd0042339875c59a59b0820a46ece356/greenlet-3.5.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ff251e9a0279522e62f6176412869395a64ddf2b5c5f782ff609a8216a4e662", size = 615198, upload-time = "2026-04-27T12:25:25.928Z" }, - { url = "https://files.pythonhosted.org/packages/10/80/3b2c0a895d6698f6ddb31b07942ebfa982f3e30888bc5546a5b5990de8b2/greenlet-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d874e79afd41a96e11ff4c5d0bc90a80973e476fda1c2c64985667397df432b", size = 1574927, upload-time = "2026-04-27T12:53:25.81Z" }, - { url = "https://files.pythonhosted.org/packages/44/0e/f354af514a4c61454dbc68e44d47544a5a4d6317e30b77ddfa3a09f4c5f3/greenlet-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0ed006e4b86c59de7467eb2601cd1b77b5a7d657d1ee55e30fe30d76451edba4", size = 1642683, upload-time = "2026-04-27T12:25:23.9Z" }, - { url = "https://files.pythonhosted.org/packages/fa/6a/87f38255201e993a1915265ebb80cd7c2c78b04a45744995abbf6b259fd8/greenlet-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:703cb211b820dbffbbc55a16bfc6e4583a6e6e990f33a119d2cc8b83211119c8", size = 238115, upload-time = "2026-04-27T12:21:48.845Z" }, - { url = "https://files.pythonhosted.org/packages/e3/f8/450fe3c5938fa737ea4d22699772e6e34e8e24431a47bf4e8a1ceed4a98e/greenlet-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:6c18dfb59c70f5a94acd271c72e90128c3c776e41e5f07767908c8c1b74ad339", size = 235017, upload-time = "2026-04-27T12:22:26.768Z" }, - { url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228, upload-time = "2026-04-27T12:20:34.421Z" }, - { url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775, upload-time = "2026-04-27T12:52:41.045Z" }, - { url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436, upload-time = "2026-04-27T12:59:41.595Z" }, - { url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388, upload-time = "2026-04-27T12:25:28.008Z" }, - { url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768, upload-time = "2026-04-27T12:53:28.099Z" }, - { url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983, upload-time = "2026-04-27T12:25:26.903Z" }, - { url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840, upload-time = "2026-04-27T12:23:54.806Z" }, - { url = "https://files.pythonhosted.org/packages/cb/cb/baa584cb00532126ffe12d9787db0a60c5a4f55c27bfe2666df5d4c30a32/greenlet-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:83ed9f27f1680b50e89f40f6df348a290ea234b249a4003d366663a12eab94f2", size = 235615, upload-time = "2026-04-27T12:21:38.57Z" }, - { url = "https://files.pythonhosted.org/packages/0c/58/fc576f99037ce19c5aa16628e4c3226b6d1419f72a62c79f5f40576e6eb3/greenlet-3.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:5a5ed18de6a0f6cc7087f1563f6bd93fc7df1c19165ca01e9bde5a5dc281d106", size = 285066, upload-time = "2026-04-27T12:23:05.033Z" }, - { url = "https://files.pythonhosted.org/packages/4a/ba/b28ddbe6bfad6a8ac196ef0e8cff37bc65b79735995b9e410923fffeeb70/greenlet-3.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a717fbc46d8a354fa675f7c1e813485b6ba3885f9bef0cd56e5ba27d758ff5b", size = 604414, upload-time = "2026-04-27T12:52:42.358Z" }, - { url = "https://files.pythonhosted.org/packages/09/06/4b69f8f0b67603a8be2790e55107a190b376f2627fe0eaf5695d85ffb3cd/greenlet-3.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddc090c5c1792b10246a78e8c2163ebbe04cf877f9d785c230a7b27b39ad038e", size = 617349, upload-time = "2026-04-27T12:59:43.32Z" }, - { url = "https://files.pythonhosted.org/packages/8a/17/a3918541fd0ddefe024a69de6d16aa7b46d36ac19562adaa63c7fa180eff/greenlet-3.5.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2094acd54b272cb6eae8c03dd87b3fa1820a4cef18d6889c378d503500a1dc13", size = 613927, upload-time = "2026-04-27T12:25:30.28Z" }, - { url = "https://files.pythonhosted.org/packages/ee/e1/bd0af6213c7dd33175d8a462d4c1fe1175124ebed4855bc1475a5b5242c2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5e05ba267789ea87b5a155cf0e810b1ab88bf18e9e8740813945ceb8ee4350ba", size = 1570893, upload-time = "2026-04-27T12:53:29.483Z" }, - { url = "https://files.pythonhosted.org/packages/9b/2a/0789702f864f5382cb476b93d7a9c823c10472658102ccd65f415747d2e2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0ecec963079cd58cbd14723582384f11f166fd58883c15dcbfb342e0bc9b5846", size = 1636060, upload-time = "2026-04-27T12:25:28.845Z" }, - { url = "https://files.pythonhosted.org/packages/b2/8f/22bf9df92bbff0eb07842b60f7e63bf7675a9742df628437a9f02d09137f/greenlet-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:728d9667d8f2f586644b748dbd9bb67e50d6a9381767d1357714ea6825bb3bf5", size = 238740, upload-time = "2026-04-27T12:24:01.341Z" }, - { url = "https://files.pythonhosted.org/packages/b6/b7/9c5c3d653bd4ff614277c049ac676422e2c557db47b4fe43e6313fc005dc/greenlet-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:47422135b1d308c14b2c6e758beedb1acd33bb91679f5670edf77bf46244722b", size = 235525, upload-time = "2026-04-27T12:23:12.308Z" }, - { url = "https://files.pythonhosted.org/packages/94/5e/a70f31e3e8d961c4ce589c15b28e4225d63704e431a23932a3808cbcc867/greenlet-3.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:f35807464c4c58c55f0d31dfa83c541a5615d825c2fe3d2b95360cf7c4e3c0a8", size = 285564, upload-time = "2026-04-27T12:23:08.555Z" }, - { url = "https://files.pythonhosted.org/packages/af/a6/046c0a28e21833e4086918218cfb3d8bed51c075a1b700f20b9d7861c0f4/greenlet-3.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55fa7ea52771be44af0de27d8b80c02cd18c2c3cddde6c847ecebdf72418b6a1", size = 651166, upload-time = "2026-04-27T12:52:43.644Z" }, - { url = "https://files.pythonhosted.org/packages/47/f8/4af27f71c5ff32a7fbc516adb46370d9c4ae2bc7bd3dc7d066ac542b4b15/greenlet-3.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a97e4821aa710603f94de0da25f25096454d78ffdace5dc77f3a006bc01abba3", size = 663792, upload-time = "2026-04-27T12:59:44.93Z" }, - { url = "https://files.pythonhosted.org/packages/a3/59/1bd6d7428d6ed9106efbb8c52310c60fd04f6672490f452aeaa3829aa436/greenlet-3.5.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f52a464e4ed91780bdfbbdd2b97197f3accaa629b98c200f4dffada759f3ae7", size = 660933, upload-time = "2026-04-27T12:25:33.276Z" }, - { url = "https://files.pythonhosted.org/packages/83/e4/b903e5a5fae1e8a28cdd32a0cfbfd560b668c25b692f67768822ddc5f40f/greenlet-3.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:762612baf1161ccb8437c0161c668a688223cba28e1bf038f4eb47b13e39ccdf", size = 1618401, upload-time = "2026-04-27T12:53:31.062Z" }, - { url = "https://files.pythonhosted.org/packages/0e/e3/5ec408a329acb854fb607a122e1ee5fb3ff649f9a97952948a90803c0d8e/greenlet-3.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:57a43c6079a89713522bc4bcb9f75070ecf5d3dbad7792bfe42239362cbf2a16", size = 1682038, upload-time = "2026-04-27T12:25:31.838Z" }, - { url = "https://files.pythonhosted.org/packages/91/20/6b165108058767ee643c55c5c4904d591a830ee2b3c7dbd359828fbc829f/greenlet-3.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bc59be3945ae9750b9e7d45067d01ae3fe90ea5f9ade99239dabdd6e28a5033", size = 239835, upload-time = "2026-04-27T12:24:54.136Z" }, - { url = "https://files.pythonhosted.org/packages/4e/62/1c498375cee177b55d980c1db319f26470e5309e54698c8f8fc06c0fd539/greenlet-3.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:a96fcee45e03fe30a62669fd16ab5c9d3c172660d3085605cb1e2d1280d3c988", size = 236862, upload-time = "2026-04-27T12:23:24.957Z" }, - { url = "https://files.pythonhosted.org/packages/78/a8/4522939255bb5409af4e87132f915446bf3622c2c292d14d3c38d128ae82/greenlet-3.5.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:a10a732421ab4fec934783ce3e54763470d0181db6e3468f9103a275c3ed1853", size = 293614, upload-time = "2026-04-27T12:24:12.874Z" }, - { url = "https://files.pythonhosted.org/packages/15/5e/8744c52e2c027b5a8772a01561934c8835f869733e101f62075c60430340/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fc391b1566f2907d17aaebe78f8855dc45675159a775fcf9e61f8ee0078e87f", size = 650723, upload-time = "2026-04-27T12:52:45.412Z" }, - { url = "https://files.pythonhosted.org/packages/00/ef/7b4c39c03cf46ceca512c5d3f914afd85aa30b2cc9a93015b0dd73e4be6c/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:680bd0e7ad5e8daa8a4aa89f68fd6adc834b8a8036dc256533f7e08f4a4b01f7", size = 656529, upload-time = "2026-04-27T12:59:46.295Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b5/c7768f352f5c010f92064d0063f987e7dc0cd290a6d92a34109015ce4aa1/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddb36c7d6c9c0a65f18c7258634e0c416c6ab59caac8c987b96f80c2ebda0112", size = 654364, upload-time = "2026-04-27T12:25:35.64Z" }, - { url = "https://files.pythonhosted.org/packages/ef/d0/079ebe12e4b1fc758857ce5be1a5e73f06870f2101e52611d1e71925ce54/greenlet-3.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e5ddf316ced87539144621453c3aef229575825fe60c604e62bedc4003f372b2", size = 1614204, upload-time = "2026-04-27T12:53:32.618Z" }, - { url = "https://files.pythonhosted.org/packages/6d/89/6c2fb63df3596552d20e58fb4d96669243388cf680cff222758812c7bfaa/greenlet-3.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4a448128607be0de65342dc9b31be7f948ef4cc0bc8832069350abefd310a8f2", size = 1675480, upload-time = "2026-04-27T12:25:34.168Z" }, - { url = "https://files.pythonhosted.org/packages/15/32/77ee8a6c1564fc345a491a4e85b3bf360e4cf26eac98c4532d2fdb96e01f/greenlet-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d60097128cb0a1cab9ea541186ea13cd7b847b8449a7787c2e2350da0cb82d86", size = 245324, upload-time = "2026-04-27T12:24:40.295Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "hf-xet" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, - { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, - { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, - { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, - { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, - { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, - { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, - { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, - { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, - { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, - { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, - { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, - { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, - { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[[package]] -name = "huggingface-hub" -version = "1.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "httpx" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c6/ae/222a91937ebee7f62c0ca8f5ee0afd97577caf24c0abb927d1f5c7e9f6d2/huggingface_hub-1.28.0.tar.gz", hash = "sha256:46a2e950c09234de54093d587d1675382f0d08dbd600d9fb599b5932f5b2c6cb", size = 959609, upload-time = "2026-08-18T12:27:15.101Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, -] - [[package]] name = "idna" version = "3.15" @@ -857,23 +667,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/2b/5c8ad37807efaedb14b20f01f36462684468c80da5b74f4018fb4c1804b5/pyiceberg-0.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:eb3a0a3e630ee89758eb96b39b456f4697732351fb0c080e9498ea578f9b71f9", size = 530923, upload-time = "2026-03-03T00:10:26.196Z" }, ] -[package.optional-dependencies] -sql-sqlite = [ - { name = "sqlalchemy" }, -] - [[package]] name = "pyiceberg-core" source = { editable = "." } [package.dev-dependencies] dev = [ - { name = "datafusion" }, { name = "fastavro" }, - { name = "huggingface-hub" }, { name = "maturin" }, { name = "pyarrow" }, - { name = "pyiceberg", extra = ["sql-sqlite"] }, + { name = "pyiceberg" }, { name = "pytest" }, ] @@ -881,12 +684,10 @@ dev = [ [package.metadata.requires-dev] dev = [ - { name = "datafusion", specifier = "==54.*" }, { name = "fastavro", specifier = ">=1.11.1" }, - { name = "huggingface-hub", specifier = ">=0.20" }, { name = "maturin", specifier = ">=1.9.4,<2.0" }, { name = "pyarrow", specifier = ">=17" }, - { name = "pyiceberg", extras = ["sql-sqlite"], specifier = ">=0.11" }, + { name = "pyiceberg", specifier = ">=0.11" }, { name = "pytest", specifier = ">=8.3.2" }, ] @@ -1009,70 +810,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] -[[package]] -name = "pyyaml" -version = "6.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, - { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, - { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, - { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, - { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, - { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, - { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, - { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, - { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, - { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, - { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, - { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, - { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, - { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, - { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, - { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, - { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, - { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, - { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, - { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, - { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, - { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, - { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, - { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, - { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, - { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, - { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, - { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, - { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, - { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, - { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, - { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, - { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, - { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, - { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, - { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, -] - [[package]] name = "requests" version = "2.33.1" @@ -1110,66 +847,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "sqlalchemy" -version = "2.0.49" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/76/f908955139842c362aa877848f42f9249642d5b69e06cee9eae5111da1bd/sqlalchemy-2.0.49-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:42e8804962f9e6f4be2cbaedc0c3718f08f60a16910fa3d86da5a1e3b1bfe60f", size = 2159321, upload-time = "2026-04-03T16:50:11.8Z" }, - { url = "https://files.pythonhosted.org/packages/24/e2/17ba0b7bfbd8de67196889b6d951de269e8a46057d92baca162889beb16d/sqlalchemy-2.0.49-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc992c6ed024c8c3c592c5fc9846a03dd68a425674900c70122c77ea16c5fb0b", size = 3238937, upload-time = "2026-04-03T16:54:45.731Z" }, - { url = "https://files.pythonhosted.org/packages/90/1e/410dd499c039deacff395eec01a9da057125fcd0c97e3badc252c6a2d6a7/sqlalchemy-2.0.49-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6eb188b84269f357669b62cb576b5b918de10fb7c728a005fa0ebb0b758adce1", size = 3237188, upload-time = "2026-04-03T16:56:53.217Z" }, - { url = "https://files.pythonhosted.org/packages/ab/06/e797a8b98a3993ac4bc785309b9b6d005457fc70238ee6cefa7c8867a92e/sqlalchemy-2.0.49-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:62557958002b69699bdb7f5137c6714ca1133f045f97b3903964f47db97ea339", size = 3190061, upload-time = "2026-04-03T16:54:47.489Z" }, - { url = "https://files.pythonhosted.org/packages/44/d3/5a9f7ef580af1031184b38235da6ac58c3b571df01c9ec061c44b2b0c5a6/sqlalchemy-2.0.49-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:da9b91bca419dc9b9267ffadde24eae9b1a6bffcd09d0a207e5e3af99a03ce0d", size = 3211477, upload-time = "2026-04-03T16:56:55.056Z" }, - { url = "https://files.pythonhosted.org/packages/69/ec/7be8c8cb35f038e963a203e4fe5a028989167cc7299927b7cf297c271e37/sqlalchemy-2.0.49-cp310-cp310-win32.whl", hash = "sha256:5e61abbec255be7b122aa461021daa7c3f310f3e743411a67079f9b3cc91ece3", size = 2119965, upload-time = "2026-04-03T17:00:50.009Z" }, - { url = "https://files.pythonhosted.org/packages/b5/31/0defb93e3a10b0cf7d1271aedd87251a08c3a597ee4f353281769b547b5a/sqlalchemy-2.0.49-cp310-cp310-win_amd64.whl", hash = "sha256:0c98c59075b890df8abfcc6ad632879540f5791c68baebacb4f833713b510e75", size = 2142935, upload-time = "2026-04-03T17:00:51.675Z" }, - { url = "https://files.pythonhosted.org/packages/60/b5/e3617cc67420f8f403efebd7b043128f94775e57e5b84e7255203390ceae/sqlalchemy-2.0.49-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5070135e1b7409c4161133aa525419b0062088ed77c92b1da95366ec5cbebbe", size = 2159126, upload-time = "2026-04-03T16:50:13.242Z" }, - { url = "https://files.pythonhosted.org/packages/20/9b/91ca80403b17cd389622a642699e5f6564096b698e7cdcbcbb6409898bc4/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ac7a3e245fd0310fd31495eb61af772e637bdf7d88ee81e7f10a3f271bff014", size = 3315509, upload-time = "2026-04-03T16:54:49.332Z" }, - { url = "https://files.pythonhosted.org/packages/b1/61/0722511d98c54de95acb327824cb759e8653789af2b1944ab1cc69d32565/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d4e5a0ceba319942fa6b585cf82539288a61e314ef006c1209f734551ab9536", size = 3315014, upload-time = "2026-04-03T16:56:56.376Z" }, - { url = "https://files.pythonhosted.org/packages/46/55/d514a653ffeb4cebf4b54c47bec32ee28ad89d39fafba16eeed1d81dccd5/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3ddcb27fb39171de36e207600116ac9dfd4ae46f86c82a9bf3934043e80ebb88", size = 3267388, upload-time = "2026-04-03T16:54:51.272Z" }, - { url = "https://files.pythonhosted.org/packages/2f/16/0dcc56cb6d3335c1671a2258f5d2cb8267c9a2260e27fde53cbfb1b3540a/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:32fe6a41ad97302db2931f05bb91abbcc65b5ce4c675cd44b972428dd2947700", size = 3289602, upload-time = "2026-04-03T16:56:57.63Z" }, - { url = "https://files.pythonhosted.org/packages/51/6c/f8ab6fb04470a133cd80608db40aa292e6bae5f162c3a3d4ab19544a67af/sqlalchemy-2.0.49-cp311-cp311-win32.whl", hash = "sha256:46d51518d53edfbe0563662c96954dc8fcace9832332b914375f45a99b77cc9a", size = 2119044, upload-time = "2026-04-03T17:00:53.455Z" }, - { url = "https://files.pythonhosted.org/packages/c4/59/55a6d627d04b6ebb290693681d7683c7da001eddf90b60cfcc41ee907978/sqlalchemy-2.0.49-cp311-cp311-win_amd64.whl", hash = "sha256:951d4a210744813be63019f3df343bf233b7432aadf0db54c75802247330d3af", size = 2143642, upload-time = "2026-04-03T17:00:54.769Z" }, - { url = "https://files.pythonhosted.org/packages/49/b3/2de412451330756aaaa72d27131db6dde23995efe62c941184e15242a5fa/sqlalchemy-2.0.49-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbccb45260e4ff1b7db0be80a9025bb1e6698bdb808b83fff0000f7a90b2c0b", size = 2157681, upload-time = "2026-04-03T16:53:07.132Z" }, - { url = "https://files.pythonhosted.org/packages/50/84/b2a56e2105bd11ebf9f0b93abddd748e1a78d592819099359aa98134a8bf/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb37f15714ec2652d574f021d479e78cd4eb9d04396dca36568fdfffb3487982", size = 3338976, upload-time = "2026-04-03T17:07:40Z" }, - { url = "https://files.pythonhosted.org/packages/2c/fa/65fcae2ed62f84ab72cf89536c7c3217a156e71a2c111b1305ab6f0690e2/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb9ec6436a820a4c006aad1ac351f12de2f2dbdaad171692ee457a02429b672", size = 3351937, upload-time = "2026-04-03T17:12:23.374Z" }, - { url = "https://files.pythonhosted.org/packages/f8/2f/6fd118563572a7fe475925742eb6b3443b2250e346a0cc27d8d408e73773/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8d6efc136f44a7e8bc8088507eaabbb8c2b55b3dbb63fe102c690da0ddebe55e", size = 3281646, upload-time = "2026-04-03T17:07:41.949Z" }, - { url = "https://files.pythonhosted.org/packages/c5/d7/410f4a007c65275b9cf82354adb4bb8ba587b176d0a6ee99caa16fe638f8/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e06e617e3d4fd9e51d385dfe45b077a41e9d1b033a7702551e3278ac597dc750", size = 3316695, upload-time = "2026-04-03T17:12:25.642Z" }, - { url = "https://files.pythonhosted.org/packages/d9/95/81f594aa60ded13273a844539041ccf1e66c5a7bed0a8e27810a3b52d522/sqlalchemy-2.0.49-cp312-cp312-win32.whl", hash = "sha256:83101a6930332b87653886c01d1ee7e294b1fe46a07dd9a2d2b4f91bcc88eec0", size = 2117483, upload-time = "2026-04-03T17:05:40.896Z" }, - { url = "https://files.pythonhosted.org/packages/47/9e/fd90114059175cac64e4fafa9bf3ac20584384d66de40793ae2e2f26f3bb/sqlalchemy-2.0.49-cp312-cp312-win_amd64.whl", hash = "sha256:618a308215b6cececb6240b9abde545e3acdabac7ae3e1d4e666896bf5ba44b4", size = 2144494, upload-time = "2026-04-03T17:05:42.282Z" }, - { url = "https://files.pythonhosted.org/packages/ae/81/81755f50eb2478eaf2049728491d4ea4f416c1eb013338682173259efa09/sqlalchemy-2.0.49-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df2d441bacf97022e81ad047e1597552eb3f83ca8a8f1a1fdd43cd7fe3898120", size = 2154547, upload-time = "2026-04-03T16:53:08.64Z" }, - { url = "https://files.pythonhosted.org/packages/a2/bc/3494270da80811d08bcfa247404292428c4fe16294932bce5593f215cad9/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e20e511dc15265fb433571391ba313e10dd8ea7e509d51686a51313b4ac01a2", size = 3280782, upload-time = "2026-04-03T17:07:43.508Z" }, - { url = "https://files.pythonhosted.org/packages/cd/f5/038741f5e747a5f6ea3e72487211579d8cbea5eb9827a9cbd61d0108c4bd/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47604cb2159f8bbd5a1ab48a714557156320f20871ee64d550d8bf2683d980d3", size = 3297156, upload-time = "2026-04-03T17:12:27.697Z" }, - { url = "https://files.pythonhosted.org/packages/88/50/a6af0ff9dc954b43a65ca9b5367334e45d99684c90a3d3413fc19a02d43c/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:22d8798819f86720bc646ab015baff5ea4c971d68121cb36e2ebc2ee43ead2b7", size = 3228832, upload-time = "2026-04-03T17:07:45.38Z" }, - { url = "https://files.pythonhosted.org/packages/bc/d1/5f6bdad8de0bf546fc74370939621396515e0cdb9067402d6ba1b8afbe9a/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9b1c058c171b739e7c330760044803099c7fff11511e3ab3573e5327116a9c33", size = 3267000, upload-time = "2026-04-03T17:12:29.657Z" }, - { url = "https://files.pythonhosted.org/packages/f7/30/ad62227b4a9819a5e1c6abff77c0f614fa7c9326e5a3bdbee90f7139382b/sqlalchemy-2.0.49-cp313-cp313-win32.whl", hash = "sha256:a143af2ea6672f2af3f44ed8f9cd020e9cc34c56f0e8db12019d5d9ecf41cb3b", size = 2115641, upload-time = "2026-04-03T17:05:43.989Z" }, - { url = "https://files.pythonhosted.org/packages/17/3a/7215b1b7d6d49dc9a87211be44562077f5f04f9bb5a59552c1c8e2d98173/sqlalchemy-2.0.49-cp313-cp313-win_amd64.whl", hash = "sha256:12b04d1db2663b421fe072d638a138460a51d5a862403295671c4f3987fb9148", size = 2141498, upload-time = "2026-04-03T17:05:45.7Z" }, - { url = "https://files.pythonhosted.org/packages/28/4b/52a0cb2687a9cd1648252bb257be5a1ba2c2ded20ba695c65756a55a15a4/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24bd94bb301ec672d8f0623eba9226cc90d775d25a0c92b5f8e4965d7f3a1518", size = 3560807, upload-time = "2026-04-03T16:58:31.666Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d8/fda95459204877eed0458550d6c7c64c98cc50c2d8d618026737de9ed41a/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51d3db74ba489266ef55c7a4534eb0b8db9a326553df481c11e5d7660c8364d", size = 3527481, upload-time = "2026-04-03T17:06:00.155Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0a/2aac8b78ac6487240cf7afef8f203ca783e8796002dc0cf65c4ee99ff8bb/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:55250fe61d6ebfd6934a272ee16ef1244e0f16b7af6cd18ab5b1fc9f08631db0", size = 3468565, upload-time = "2026-04-03T16:58:33.414Z" }, - { url = "https://files.pythonhosted.org/packages/a5/3d/ce71cfa82c50a373fd2148b3c870be05027155ce791dc9a5dcf439790b8b/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:46796877b47034b559a593d7e4b549aba151dae73f9e78212a3478161c12ab08", size = 3477769, upload-time = "2026-04-03T17:06:02.787Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e8/0a9f5c1f7c6f9ca480319bf57c2d7423f08d31445974167a27d14483c948/sqlalchemy-2.0.49-cp313-cp313t-win32.whl", hash = "sha256:9c4969a86e41454f2858256c39bdfb966a20961e9b58bf8749b65abf447e9a8d", size = 2143319, upload-time = "2026-04-03T17:02:04.328Z" }, - { url = "https://files.pythonhosted.org/packages/0e/51/fb5240729fbec73006e137c4f7a7918ffd583ab08921e6ff81a999d6517a/sqlalchemy-2.0.49-cp313-cp313t-win_amd64.whl", hash = "sha256:b9870d15ef00e4d0559ae10ee5bc71b654d1f20076dbe8bc7ed19b4c0625ceba", size = 2175104, upload-time = "2026-04-03T17:02:05.989Z" }, - { url = "https://files.pythonhosted.org/packages/55/33/bf28f618c0a9597d14e0b9ee7d1e0622faff738d44fe986ee287cdf1b8d0/sqlalchemy-2.0.49-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:233088b4b99ebcbc5258c755a097aa52fbf90727a03a5a80781c4b9c54347a2e", size = 2156356, upload-time = "2026-04-03T16:53:09.914Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a7/5f476227576cb8644650eff68cc35fa837d3802b997465c96b8340ced1e2/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57ca426a48eb2c682dae8204cd89ea8ab7031e2675120a47924fabc7caacbc2a", size = 3276486, upload-time = "2026-04-03T17:07:46.9Z" }, - { url = "https://files.pythonhosted.org/packages/2e/84/efc7c0bf3a1c5eef81d397f6fddac855becdbb11cb38ff957888603014a7/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685e93e9c8f399b0c96a624799820176312f5ceef958c0f88215af4013d29066", size = 3281479, upload-time = "2026-04-03T17:12:32.226Z" }, - { url = "https://files.pythonhosted.org/packages/91/68/bb406fa4257099c67bd75f3f2261b129c63204b9155de0d450b37f004698/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e0400fa22f79acc334d9a6b185dc00a44a8e6578aa7e12d0ddcd8434152b187", size = 3226269, upload-time = "2026-04-03T17:07:48.678Z" }, - { url = "https://files.pythonhosted.org/packages/67/84/acb56c00cca9f251f437cb49e718e14f7687505749ea9255d7bd8158a6df/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a05977bffe9bffd2229f477fa75eabe3192b1b05f408961d1bebff8d1cd4d401", size = 3248260, upload-time = "2026-04-03T17:12:34.381Z" }, - { url = "https://files.pythonhosted.org/packages/56/19/6a20ea25606d1efd7bd1862149bb2a22d1451c3f851d23d887969201633f/sqlalchemy-2.0.49-cp314-cp314-win32.whl", hash = "sha256:0f2fa354ba106eafff2c14b0cc51f22801d1e8b2e4149342023bd6f0955de5f5", size = 2118463, upload-time = "2026-04-03T17:05:47.093Z" }, - { url = "https://files.pythonhosted.org/packages/cf/4f/8297e4ed88e80baa1f5aa3c484a0ee29ef3c69c7582f206c916973b75057/sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl", hash = "sha256:77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5", size = 2144204, upload-time = "2026-04-03T17:05:48.694Z" }, - { url = "https://files.pythonhosted.org/packages/1f/33/95e7216df810c706e0cd3655a778604bbd319ed4f43333127d465a46862d/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1dc3368794d522f43914e03312202523cc89692f5389c32bea0233924f8d977", size = 3565474, upload-time = "2026-04-03T16:58:35.128Z" }, - { url = "https://files.pythonhosted.org/packages/0c/a4/ed7b18d8ccf7f954a83af6bb73866f5bc6f5636f44c7731fbb741f72cc4f/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c821c47ecfe05cc32140dcf8dc6fd5d21971c86dbd56eabfe5ba07a64910c01", size = 3530567, upload-time = "2026-04-03T17:06:04.587Z" }, - { url = "https://files.pythonhosted.org/packages/73/a3/20faa869c7e21a827c4a2a42b41353a54b0f9f5e96df5087629c306df71e/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9c04bff9a5335eb95c6ecf1c117576a0aa560def274876fd156cfe5510fccc61", size = 3474282, upload-time = "2026-04-03T16:58:37.131Z" }, - { url = "https://files.pythonhosted.org/packages/b7/50/276b9a007aa0764304ad467eceb70b04822dc32092492ee5f322d559a4dc/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7f605a456948c35260e7b2a39f8952a26f077fd25653c37740ed186b90aaa68a", size = 3480406, upload-time = "2026-04-03T17:06:07.176Z" }, - { url = "https://files.pythonhosted.org/packages/e5/c3/c80fcdb41905a2df650c2a3e0337198b6848876e63d66fe9188ef9003d24/sqlalchemy-2.0.49-cp314-cp314t-win32.whl", hash = "sha256:6270d717b11c5476b0cbb21eedc8d4dbb7d1a956fd6c15a23e96f197a6193158", size = 2149151, upload-time = "2026-04-03T17:02:07.281Z" }, - { url = "https://files.pythonhosted.org/packages/05/52/9f1a62feab6ed368aff068524ff414f26a6daebc7361861035ae00b05530/sqlalchemy-2.0.49-cp314-cp314t-win_amd64.whl", hash = "sha256:275424295f4256fd301744b8f335cff367825d270f155d522b30c7bf49903ee7", size = 2184178, upload-time = "2026-04-03T17:02:08.623Z" }, - { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, -] - [[package]] name = "strictyaml" version = "1.7.3" @@ -1245,18 +922,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] -[[package]] -name = "tqdm" -version = "4.67.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, -] - [[package]] name = "typing-extensions" version = "4.15.0" From 6fa8e03834dca17481441ec758afbbe12efc27f6 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 4 Sep 2026 03:06:27 -0700 Subject: [PATCH 28/74] refactor: materialize constant metadata columns via a ConstantLookup action (#3121) Unify the reserved/metadata column handling behind a single set of transform actions, per review feedback on #3058. Behavior is unchanged. --- .../src/arrow/record_batch_transformer.rs | 133 +++++++++++------- crates/iceberg/src/arrow/value.rs | 28 ++-- 2 files changed, 100 insertions(+), 61 deletions(-) diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index b06c656e54..800dc6da41 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -150,35 +150,36 @@ pub(crate) enum ColumnSource { source_index: usize, }, - // Signifies that a new column has been inserted before the column - // with index `index`. (we choose "before" rather than "after" so - // that we can use usize; if we insert after, then we need to - // be able to store -1 here to signify that a new - // column is to be added at the front of the column list). - // If multiple columns need to be inserted at a given - // location, they should all be given the same index, as the index - // here refers to the original RecordBatch, not the interim state after - // a preceding operation. - Add { + // Materializes a column by looking up its per-file constant in the + // transformer's `constant_fields` map by `field_id`, producing a column of + // `target_type`. Handles the three non-coalesce `ColumnConstant` variants: + // scalar values (`_file`, `_spec_id`, identity partitions), the `_partition` + // struct, and all-null metadata columns. The coalesce variant routes to + // `Coalesce` instead. Resolved at apply time rather than baking the value here. + ConstantLookup { + field_id: i32, target_type: DataType, - value: Option, }, - // A struct column where each child is a constant primitive value. - // Used for the _partition metadata column. - AddStructConstant { - fields: Fields, - child_values: Vec>, + // Inserts a schema-evolution default value (the field's `initial_default`, + // or null for an absent optional field). Distinct from `ConstantLookup`: + // this value comes from the table schema, not the per-file constant map. + Add { + target_type: DataType, + value: Option, }, // A metadata column read from the file and coalesced with a per-file - // fallback: the source column's value where non-null, else `fallback`. - // Used for `_last_updated_sequence_number` when the file physically carries - // the per-row column. The result is cast to `target_type` so it matches the - // (run-end-encoded) type the constant and null paths produce for the column. - CoalesceLastUpdatedSeq { + // fallback looked up in `constant_fields` by `field_id`: the source column's + // value where non-null, else the fallback constant. The entry at `field_id` + // must be a `ColumnConstant::CoalesceLastUpdatedSeq` (the apply arm rejects + // any other variant). Used for `_last_updated_sequence_number` when the file + // physically carries the per-row column. The result is cast to `target_type` + // so it matches the (run-end-encoded) type the constant and null paths + // produce for the column. + Coalesce { + field_id: i32, source_index: usize, - fallback: PrimitiveLiteral, target_type: DataType, }, // The iceberg spec refers to other permissible schema evolution actions @@ -229,8 +230,9 @@ enum SchemaComparison { /// /// All per-file constants (scalar metadata like `_file`, identity partition values, /// and the `_partition` struct) are stored in a single `constant_fields` map keyed by -/// field_id. This unified representation (via [`ColumnConstant`]) means the -/// transformer handles all constant columns through one code path. +/// field_id (via [`ColumnConstant`]). Each is resolved at apply time from that single +/// map (through a `ConstantLookup` or `Coalesce` action) rather than baked into the +/// action. #[derive(Debug)] pub(crate) struct RecordBatchTransformerBuilder { snapshot_schema: Arc, @@ -704,9 +706,12 @@ impl RecordBatchTransformer { Some(ColumnConstant::Struct(pc)) if *field_id == RESERVED_FIELD_ID_PARTITION => { - return Ok(ColumnSource::AddStructConstant { - fields: pc.fields().clone(), - child_values: pc.child_values().to_vec(), + // The struct apply arm rebuilds the column from the map entry's own + // fields/child_values, so `target_type` is a placeholder here and is + // not read for the struct case (only the scalar/null cases consult it). + return Ok(ColumnSource::ConstantLookup { + field_id: *field_id, + target_type: DataType::Struct(pc.fields().clone()), }); } Some(ColumnConstant::Struct(_)) => { @@ -747,8 +752,8 @@ impl RecordBatchTransformer { .clone() }; - return Ok(ColumnSource::Add { - value: Some(datum.literal().clone()), + return Ok(ColumnSource::ConstantLookup { + field_id: *field_id, target_type: arrow_type, }); } @@ -756,8 +761,8 @@ impl RecordBatchTransformer { // to read from the file instead of using the constant. } Some(ColumnConstant::Null(arrow_type)) => { - return Ok(ColumnSource::Add { - value: None, + return Ok(ColumnSource::ConstantLookup { + field_id: *field_id, target_type: arrow_type.clone(), }); } @@ -775,9 +780,9 @@ impl RecordBatchTransformer { ), ) })?; - return Ok(ColumnSource::CoalesceLastUpdatedSeq { + return Ok(ColumnSource::Coalesce { + field_id: *field_id, source_index: *source_index, - fallback: datum.literal().clone(), target_type: datum_to_arrow_type_with_ree(datum), }); } @@ -928,23 +933,57 @@ impl RecordBatchTransformer { } => cast(&*columns[*source_index], target_type)?, ColumnSource::Add { target_type, value } => { - Self::create_column(target_type, value, num_rows)? + Self::create_column(target_type, value.as_ref(), num_rows)? } - ColumnSource::AddStructConstant { - fields, - child_values, - } => Self::create_struct_column(fields, child_values, num_rows)?, + // Resolve the per-file constant for this field id from the map and + // materialize it as the target type. + ColumnSource::ConstantLookup { + field_id, + target_type, + } => match self.constant_fields.get(field_id) { + Some(ColumnConstant::Scalar(datum)) => { + Self::create_column(target_type, Some(datum.literal()), num_rows)? + } + Some(ColumnConstant::Null(_)) => { + Self::create_column(target_type, None, num_rows)? + } + Some(ColumnConstant::Struct(sc)) => { + Self::create_struct_column(sc.fields(), sc.child_values(), num_rows)? + } + other => { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "ConstantLookup for field id {field_id} expected a scalar, \ + null, or struct constant, found {other:?}" + ), + )); + } + }, - ColumnSource::CoalesceLastUpdatedSeq { + ColumnSource::Coalesce { + field_id, source_index, - fallback, - target_type, - } => Self::create_coalesce_column( - &columns[*source_index], - fallback, target_type, - )?, + } => { + let Some(ColumnConstant::CoalesceLastUpdatedSeq(datum)) = + self.constant_fields.get(field_id) + else { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Coalesce for field id {field_id} expected a coalesce \ + fallback constant in the constant map" + ), + )); + }; + Self::create_coalesce_column( + &columns[*source_index], + datum.literal(), + target_type, + )? + } }) }) .collect() @@ -981,7 +1020,7 @@ impl RecordBatchTransformer { fn create_column( target_type: &DataType, - prim_lit: &Option, + prim_lit: Option<&PrimitiveLiteral>, num_rows: usize, ) -> Result { // Check if this is a RunEndEncoded type (for constant fields) @@ -1033,7 +1072,7 @@ impl RecordBatchTransformer { .iter() .zip(child_values.iter()) .map(|(field, value)| { - create_primitive_array_repeated(field.data_type(), value, num_rows) + create_primitive_array_repeated(field.data_type(), value.as_ref(), num_rows) }) .collect::>()?; diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index 34b4336ef2..13733559c8 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -633,7 +633,7 @@ pub fn arrow_primitive_to_literal( /// a single value that represents all rows. pub(crate) fn create_primitive_array_single_element( data_type: &DataType, - prim_lit: &Option, + prim_lit: Option<&PrimitiveLiteral>, ) -> Result { match (data_type, prim_lit) { (DataType::Boolean, Some(PrimitiveLiteral::Boolean(v))) => { @@ -816,7 +816,7 @@ pub(crate) fn create_primitive_array_single_element( /// repeated for each row. pub(crate) fn create_primitive_array_repeated( data_type: &DataType, - prim_lit: &Option, + prim_lit: Option<&PrimitiveLiteral>, num_rows: usize, ) -> Result { Ok(match (data_type, prim_lit) { @@ -949,7 +949,7 @@ pub(crate) fn create_primitive_array_repeated( // Create a StructArray filled with nulls, recursively creating null children let null_arrays: Vec = fields .iter() - .map(|field| create_primitive_array_repeated(field.data_type(), &None, num_rows)) + .map(|field| create_primitive_array_repeated(field.data_type(), None, num_rows)) .collect::>>()?; Arc::new(StructArray::new( @@ -1869,7 +1869,7 @@ mod test { let target_type = DataType::Decimal128(target_precision, target_scale); let value = PrimitiveLiteral::Int128(10000000000); - let array = create_primitive_array_single_element(&target_type, &Some(value)) + let array = create_primitive_array_single_element(&target_type, Some(&value)) .expect("Failed to create decimal array"); match array.data_type() { @@ -1890,7 +1890,7 @@ mod test { let value = PrimitiveLiteral::Int128(10000000000); let num_rows = 5; - let array = create_primitive_array_repeated(&target_type, &Some(value), num_rows) + let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows) .expect("Failed to create repeated decimal array"); match array.data_type() { @@ -1910,7 +1910,7 @@ mod test { let value = PrimitiveLiteral::Long(1_740_600_000_000_000); let num_rows = 3; - let array = create_primitive_array_repeated(&target_type, &Some(value), num_rows) + let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows) .expect("Failed to create repeated timestamp microsecond array"); assert_eq!(array.data_type(), &target_type); @@ -1923,7 +1923,7 @@ mod test { let value = PrimitiveLiteral::Long(1_740_600_000_000_000); let num_rows = 2; - let array = create_primitive_array_repeated(&target_type, &Some(value), num_rows) + let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows) .expect("Failed to create repeated timestamp microsecond array with timezone"); assert_eq!(array.data_type(), &target_type); @@ -1938,7 +1938,7 @@ mod test { let utf8 = create_primitive_array_repeated( &DataType::Utf8, - &Some(PrimitiveLiteral::String(text.to_string())), + Some(&PrimitiveLiteral::String(text.to_string())), num_rows, ) .unwrap(); @@ -1948,7 +1948,7 @@ mod test { let binary = create_primitive_array_repeated( &DataType::Binary, - &Some(PrimitiveLiteral::Binary(bytes.clone())), + Some(&PrimitiveLiteral::Binary(bytes.clone())), num_rows, ) .unwrap(); @@ -1958,7 +1958,7 @@ mod test { let large = create_primitive_array_repeated( &DataType::LargeBinary, - &Some(PrimitiveLiteral::Binary(bytes.clone())), + Some(&PrimitiveLiteral::Binary(bytes.clone())), num_rows, ) .unwrap(); @@ -1968,7 +1968,7 @@ mod test { let fixed = create_primitive_array_repeated( &DataType::FixedSizeBinary(bytes.len() as i32), - &Some(PrimitiveLiteral::Binary(bytes.clone())), + Some(&PrimitiveLiteral::Binary(bytes.clone())), num_rows, ) .unwrap(); @@ -1985,7 +1985,7 @@ mod test { // num_rows == 0 must produce an empty (not one-element) array. let array = create_primitive_array_repeated( &DataType::Utf8, - &Some(PrimitiveLiteral::String("x".to_string())), + Some(&PrimitiveLiteral::String("x".to_string())), 0, ) .unwrap(); @@ -1998,7 +1998,7 @@ mod test { // than silently producing a FixedSizeBinary array of the literal's width. let err = create_primitive_array_repeated( &DataType::FixedSizeBinary(4), - &Some(PrimitiveLiteral::Binary(vec![0x01, 0x02, 0x03])), + Some(&PrimitiveLiteral::Binary(vec![0x01, 0x02, 0x03])), 2, ) .unwrap_err(); @@ -2012,7 +2012,7 @@ mod test { // empty fixed[n] array. let result = create_primitive_array_repeated( &DataType::FixedSizeBinary(4), - &Some(PrimitiveLiteral::Binary(vec![0xDE, 0xAD, 0xBE, 0xEF])), + Some(&PrimitiveLiteral::Binary(vec![0xDE, 0xAD, 0xBE, 0xEF])), 0, ); assert!(result.is_err()); From 5468f7070bcbe660a91361e27802c3e9ba9345e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 09:41:47 +0800 Subject: [PATCH 29/74] chore(deps): Bump apache/skywalking-eyes/header from 0.8.0 to 0.9.0 (#3157) Bumps [apache/skywalking-eyes/header](https://github.com/apache/skywalking-eyes) from 0.8.0 to 0.9.0. - [Release notes](https://github.com/apache/skywalking-eyes/releases) - [Changelog](https://github.com/apache/skywalking-eyes/blob/main/CHANGES.md) - [Commits](https://github.com/apache/skywalking-eyes/compare/61275cc80d0798a405cb070f7d3a8aaf7cf2c2c1...a196742f472feaffafea537ce5a2a4c3c53a8de4) --- updated-dependencies: - dependency-name: apache/skywalking-eyes/header dependency-version: 0.9.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f25e737911..435e09c205 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: tool: taplo@0.9.3,cargo-machete@0.7.0 - name: Check License Header - uses: apache/skywalking-eyes/header@61275cc80d0798a405cb070f7d3a8aaf7cf2c2c1 # v0.8.0 + uses: apache/skywalking-eyes/header@a196742f472feaffafea537ce5a2a4c3c53a8de4 # v0.9.0 - name: Check toml format run: taplo fmt --check From 63986f5ae4d6d3232181248a9d0a8796482d4a4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 09:46:57 +0800 Subject: [PATCH 30/74] chore(deps): Bump taiki-e/install-action from 2.86.6 to 2.87.2 (#3156) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.86.6 to 2.87.2. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/6cd13508893c0e7eab5f273c2575d3859bd7229a...1ed6d7be6168f6c9046541087ff549b6bc581fdf) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.87.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 435e09c205..23901fa56b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - name: Install lint tools # Keep versions in sync with the local install targets in Makefile. - uses: taiki-e/install-action@6cd13508893c0e7eab5f273c2575d3859bd7229a # v2.86.6 + uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2 with: tool: taplo@0.9.3,cargo-machete@0.7.0 @@ -212,7 +212,7 @@ jobs: - name: Install cargo-nextest if: matrix.test-suite.name == 'default' - uses: taiki-e/install-action@6cd13508893c0e7eab5f273c2575d3859bd7229a # v2.86.6 + uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2 with: tool: cargo-nextest From 6b4c27748273dd2fe48062fd62d5bb0f8e3626cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 09:47:39 +0800 Subject: [PATCH 31/74] chore(deps): Bump crate-ci/typos from 1.49.0 to 1.50.0 (#3155) Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.49.0 to 1.50.0. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/8a48f81b6c64dcfea44b3633223084c4be58ac5f...4d9c206a77c041268485162b8e2579ad7a5cb9a3) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.50.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23901fa56b..15e56b2ecc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: run: cargo machete - name: Check typos - uses: crate-ci/typos@8a48f81b6c64dcfea44b3633223084c4be58ac5f # v1.49.0 + uses: crate-ci/typos@4d9c206a77c041268485162b8e2579ad7a5cb9a3 # v1.50.0 clippy: needs: lint From 18766f5e38c405336f75c68fae1c384af84c4212 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 09:56:41 +0800 Subject: [PATCH 32/74] chore(deps): Bump rand from 0.9.5 to 0.10.2 (#3154) Bumps [rand](https://github.com/rust-random/rand) from 0.9.5 to 0.10.2. - [Release notes](https://github.com/rust-random/rand/releases) - [Changelog](https://github.com/rust-random/rand/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/rand/compare/0.9.5...0.10.2) --- updated-dependencies: - dependency-name: rand dependency-version: 0.10.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 99557339a0..bab0d84020 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7596,7 +7596,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", From cd11d72d89c1e0f5d44c24b1f985c07e83974660 Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:29:06 +0800 Subject: [PATCH 33/74] feat(scan): make FileScanTask serializable (#3091) * feat(scan): make FileScanTask serializable Replace the placeholder serde errors with a lossless representation for partition literals and use the existing serde implementations for the remaining scan context fields. Expand round-trip coverage to include a fully populated task and every primitive partition literal. [apache/iceberg-rust#3089](https://github.com/apache/iceberg-rust/issues/3089) * refactor(spec): serialize Literal directly Move the self-describing serde representation into Literal so scan task partitions use the shared values-layer implementation. Cover primitive and nested literal round trips and remove the task-local helper requested in review. * chore: update public API snapshot Record the new Serialize and Deserialize implementations for Literal and Struct. * refactor(scan): use typed FileScanTask serde adapter Serialize partition data through the existing RawLiteral implementation using the task schema and partition spec. Remove the direct Literal and Struct serde implementation and keep the public API unchanged. * refactor(scan): simplify FileScanTask serde adapter * fix(scan): handle unpartitioned task serde * refactor(scan): simplify partition serde * test(scan): split file scan task serde cases * fix(scan): harden file scan task serde * refactor(scan): delegate serde validation to builder --- crates/iceberg/public-api.txt | 2 +- crates/iceberg/src/scan/mod.rs | 216 ++++++++++++++++++++--- crates/iceberg/src/scan/task.rs | 218 +++++++++++++++++++----- crates/iceberg/src/spec/values/serde.rs | 6 + crates/iceberg/src/spec/values/tests.rs | 23 +++ 5 files changed, 401 insertions(+), 64 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 57edfd6421..ca55804966 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1306,7 +1306,7 @@ impl core::marker::StructuralPartialEq for iceberg::scan::FileScanTask impl iceberg::scan::FileScanTask pub fn iceberg::scan::FileScanTask::builder() -> FileScanTaskBuilder<((), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), ())> impl serde_core::ser::Serialize for iceberg::scan::FileScanTask -pub fn iceberg::scan::FileScanTask::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +pub fn iceberg::scan::FileScanTask::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::scan::FileScanTask pub fn iceberg::scan::FileScanTask::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::scan::FileScanTaskDeleteFile diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 2f0a876096..db70806473 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -660,12 +660,13 @@ pub mod tests { RESERVED_COL_NAME_POS, RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_DELETE_FILE_PATH, RESERVED_FIELD_ID_DELETE_FILE_POS, RESERVED_FIELD_ID_POS, }; - use crate::scan::FileScanTask; + use crate::scan::{FileScanTask, FileScanTaskDeleteFile}; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Datum, FormatVersion, Literal, MAIN_BRANCH, ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, - NestedField, Operation, PartitionSpec, PrimitiveType, Schema, Snapshot, Struct, StructType, - Summary, TableMetadata, TableMetadataBuilder, TableProperties, Type, UnboundPartitionSpec, + MappedField, NameMapping, NestedField, Operation, PartitionSpec, PrimitiveType, Schema, + Snapshot, Struct, StructType, Summary, TableMetadata, TableMetadataBuilder, + TableProperties, Transform, Type, UnboundPartitionSpec, }; use crate::table::Table; use crate::test_utils::test_runtime; @@ -2629,33 +2630,64 @@ pub mod tests { assert_eq!(string_arr.value(0), "Apache"); } - #[test] - fn test_file_scan_task_serialize_deserialize() { - let test_fn = |task: FileScanTask| { - let serialized = serde_json::to_string(&task).unwrap(); - let deserialized: FileScanTask = serde_json::from_str(&serialized).unwrap(); - - assert_eq!(task, deserialized); - }; - - // without predicate - let schema = Arc::new( + fn file_scan_task_test_schema(primitive_type: PrimitiveType) -> Arc { + Arc::new( Schema::builder() .with_fields(vec![Arc::new(NestedField::required( 1, "x", - Type::Primitive(PrimitiveType::Binary), + Type::Primitive(primitive_type), ))]) .build() .unwrap(), + ) + } + + fn assert_file_scan_task_serde_round_trip(task: FileScanTask) { + // Regression test for https://github.com/apache/iceberg-rust/issues/3089. + let serialized = serde_json::to_string(&task).unwrap(); + let deserialized: FileScanTask = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(task, deserialized); + } + + fn file_scan_task_with_partition( + primitive_type: PrimitiveType, + transform: Transform, + partition_value: Literal, + ) -> FileScanTask { + let schema = file_scan_task_test_schema(primitive_type); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .add_partition_field("x", "x_partition", transform) + .unwrap() + .build() + .unwrap(), ); + FileScanTask::builder() + .with_data_file_path("data_file_path".to_string()) + .with_file_size_in_bytes(123) + .with_start(10) + .with_length(100) + .with_project_field_ids(vec![1]) + .with_schema(schema) + .with_data_file_format(DataFileFormat::Parquet) + .with_partition(Some(Struct::from_iter([Some(partition_value)]))) + .with_partition_spec(Some(partition_spec)) + .with_case_sensitive(true) + .build() + .unwrap() + } + + #[test] + fn test_file_scan_task_serde_without_predicate() { let task = FileScanTask::builder() .with_data_file_path("data_file_path".to_string()) .with_file_size_in_bytes(0) .with_start(0) .with_length(100) .with_project_field_ids(vec![1, 2, 3]) - .with_schema(schema.clone()) + .with_schema(file_scan_task_test_schema(PrimitiveType::Binary)) .with_record_count(Some(100)) .with_first_row_id(Some(1000)) .with_data_sequence_number(Some(5)) @@ -2663,9 +2695,11 @@ pub mod tests { .with_case_sensitive(false) .build() .unwrap(); - test_fn(task); + assert_file_scan_task_serde_round_trip(task); + } - // with predicate + #[test] + fn test_file_scan_task_serde_with_predicate() { let task = FileScanTask::builder() .with_data_file_path("data_file_path".to_string()) .with_file_size_in_bytes(0) @@ -2673,12 +2707,154 @@ pub mod tests { .with_length(100) .with_project_field_ids(vec![1, 2, 3]) .with_predicate(Some(BoundPredicate::AlwaysTrue)) - .with_schema(schema) + .with_schema(file_scan_task_test_schema(PrimitiveType::Binary)) .with_data_file_format(DataFileFormat::Avro) .with_case_sensitive(false) .build() .unwrap(); - test_fn(task); + + let serialized = serde_json::to_value(&task).unwrap(); + assert!(serialized.get("record_count").is_none()); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_unpartitioned_file_scan_task_serde() { + let task = FileScanTask::builder() + .with_data_file_path("data_file_path".to_string()) + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(100) + .with_project_field_ids(vec![1, 2, 3]) + .with_schema(file_scan_task_test_schema(PrimitiveType::Binary)) + .with_data_file_format(DataFileFormat::Parquet) + .with_partition(Some(Struct::empty())) + .with_case_sensitive(false) + .build() + .unwrap(); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_all_optional_fields() { + let schema = file_scan_task_test_schema(PrimitiveType::Long); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .add_partition_field("x", "x", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let unified_partition_type = Arc::new(partition_spec.partition_type(&schema).unwrap()); + let task = FileScanTask::builder() + .with_data_file_path("data_file_path".to_string()) + .with_file_size_in_bytes(123) + .with_start(10) + .with_length(100) + .with_project_field_ids(vec![1]) + .with_schema(schema) + .with_data_file_format(DataFileFormat::Parquet) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path("delete_file_path".to_string()) + .with_file_size_in_bytes(23) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(vec![1])) + .with_referenced_data_file(Some("data_file_path".to_string())) + .with_content_offset(Some(12)) + .with_content_size_in_bytes(Some(34)) + .with_record_count(Some(5)) + .with_key_metadata(Some(vec![4, 5, 6].into_boxed_slice())) + .build(), + ]) + .with_partition(Some(Struct::from_iter([Some(Literal::long(42))]))) + .with_partition_spec(Some(partition_spec)) + .with_name_mapping(Some(Arc::new(NameMapping::new(vec![MappedField::new( + Some(1), + vec!["x".to_string()], + vec![], + )])))) + .with_unified_partition_type(Some(unified_partition_type)) + .with_case_sensitive(true) + .with_key_metadata(Some(vec![1, 2, 3].into_boxed_slice())) + .build() + .unwrap(); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_date_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::Date, + Transform::Identity, + Literal::date(19_000), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_timestamp_ns_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::TimestampNs, + Transform::Identity, + Literal::timestamp_nano(1_510_871_468_123_456_789), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_timestamptz_ns_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::TimestamptzNs, + Transform::Identity, + Literal::timestamptz_nano(1_510_871_468_123_456_789), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_decimal_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::Decimal { + precision: 9, + scale: 2, + }, + Transform::Identity, + Literal::decimal(12_345), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_uuid_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::Uuid, + Transform::Identity, + Literal::uuid(Uuid::from_u128(0x12345678_90ab_cdef_1234_567890abcdef)), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_fixed_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::Fixed(4), + Transform::Identity, + Literal::fixed([1, 2, 3, 4]), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_bucket_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::String, + Transform::Bucket(4), + Literal::int(2), + ); + assert_file_scan_task_serde_round_trip(task); } #[tokio::test] diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 3aff0f4ae6..f1a799fcc6 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use futures::stream::BoxStream; -use serde::{Deserialize, Serialize, Serializer}; +use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use crate::expr::BoundPredicate; @@ -31,26 +31,9 @@ use crate::{Error, ErrorKind, Result}; /// A stream of [`FileScanTask`]. pub type FileScanTaskStream = BoxStream<'static, Result>; -/// Serialization helper that always returns NotImplementedError. -/// Used for fields that should not be serialized but we want to be explicit about it. -fn serialize_not_implemented(_: &T, _: S) -> std::result::Result -where S: Serializer { - Err(serde::ser::Error::custom( - "Serialization not implemented for this field", - )) -} - -/// Deserialization helper that always returns NotImplementedError. -/// Used for fields that should not be deserialized but we want to be explicit about it. -fn deserialize_not_implemented<'de, D, T>(_: D) -> std::result::Result -where D: serde::Deserializer<'de> { - Err(serde::de::Error::custom( - "Deserialization not implemented for this field", - )) -} - /// A task to scan part of file. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TypedBuilder)] +#[derive(Debug, Clone, Deserialize, PartialEq, TypedBuilder)] +#[serde(try_from = "crate::scan::task::_serde::FileScanTaskSerde")] #[builder( field_defaults(setter(prefix = "with_")), build_method(into = Result) @@ -74,7 +57,6 @@ pub struct FileScanTask { /// /// Used to derive the `_row_id` metadata column: for a row without an /// explicit `_row_id`, it is this value plus the row's ordinal position. - #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] first_row_id: Option, @@ -84,7 +66,6 @@ pub struct FileScanTask { /// manifest that lacks one. /// /// Used to derive the `_last_updated_sequence_number` metadata column. - #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] data_sequence_number: Option, @@ -99,7 +80,6 @@ pub struct FileScanTask { /// The field ids to project. project_field_ids: Vec, /// The predicate to filter. - #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] predicate: Option, @@ -110,30 +90,18 @@ pub struct FileScanTask { /// Partition data from the manifest entry, used to identify which columns can use /// constant values from partition metadata vs. reading from the data file. /// Per the Iceberg spec, only identity-transformed partition fields should use constants. - #[serde(default)] - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(serialize_with = "serialize_not_implemented")] - #[serde(deserialize_with = "deserialize_not_implemented")] #[builder(default)] partition: Option, /// The partition spec for this file, used to distinguish identity transforms /// (which use partition metadata constants) from non-identity transforms like /// bucket/truncate (which must read source columns from the data file). - #[serde(default)] - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(serialize_with = "serialize_not_implemented")] - #[serde(deserialize_with = "deserialize_not_implemented")] #[builder(default)] partition_spec: Option>, /// Name mapping from table metadata (property: schema.name-mapping.default), /// used to resolve field IDs from column names when Parquet files lack field IDs /// or have field ID conflicts. - #[serde(default)] - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(serialize_with = "serialize_not_implemented")] - #[serde(deserialize_with = "deserialize_not_implemented")] #[builder(default)] name_mapping: Option>, @@ -145,11 +113,6 @@ pub struct FileScanTask { /// This is a table-level value (same for all tasks in a scan), stored per-task /// so that readers are self-contained without needing back-pointers to table /// metadata. The cost is one Arc clone per task. - /// Serde: not yet implemented (same pattern as partition, partition_spec, name_mapping). - #[serde(default)] - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(serialize_with = "serialize_not_implemented")] - #[serde(deserialize_with = "deserialize_not_implemented")] #[builder(default)] unified_partition_type: Option>, @@ -161,11 +124,9 @@ pub struct FileScanTask { /// /// Note on the trust boundary: for the standard encryption scheme this /// carries `StandardKeyMetadata`, whose payload is the *plaintext* DEK. - /// Because `FileScanTask` derives `Serialize`, that plaintext DEK is part + /// Because `FileScanTask` implements [`Serialize`], that plaintext DEK is part /// of the serialized scan plan should these tasks ever be serialized and sent /// over the network. - #[serde(default)] - #[serde(skip_serializing_if = "Option::is_none")] #[builder(default)] key_metadata: Option>, } @@ -403,6 +364,177 @@ pub struct FileScanTaskDeleteFile { pub key_metadata: Option>, } +mod _serde { + use std::sync::Arc; + + use serde::{Deserialize, Serialize}; + + use super::{FileScanTask, FileScanTaskDeleteFile}; + use crate::expr::BoundPredicate; + use crate::spec::{ + DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral, SchemaRef, StructType, + Type, + }; + use crate::{Error, ErrorKind, Result}; + + #[derive(Deserialize)] + pub(super) struct FileScanTaskSerde { + file_size_in_bytes: u64, + start: u64, + length: u64, + record_count: Option, + first_row_id: Option, + data_sequence_number: Option, + data_file_path: String, + data_file_format: DataFileFormat, + schema: SchemaRef, + project_field_ids: Vec, + predicate: Option, + deletes: Vec, + #[serde(default)] + partition: Option, + #[serde(default)] + partition_spec: Option>, + #[serde(default)] + name_mapping: Option>, + #[serde(default)] + unified_partition_type: Option>, + case_sensitive: bool, + #[serde(default)] + key_metadata: Option>, + } + + #[derive(Serialize)] + struct FileScanTaskRefSerde<'a> { + file_size_in_bytes: u64, + start: u64, + length: u64, + #[serde(skip_serializing_if = "Option::is_none")] + record_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + first_row_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + data_sequence_number: Option, + data_file_path: &'a str, + data_file_format: DataFileFormat, + schema: &'a SchemaRef, + project_field_ids: &'a [i32], + #[serde(skip_serializing_if = "Option::is_none")] + predicate: Option<&'a BoundPredicate>, + deletes: &'a [FileScanTaskDeleteFile], + #[serde(skip_serializing_if = "Option::is_none")] + partition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + partition_spec: Option<&'a Arc>, + #[serde(skip_serializing_if = "Option::is_none")] + name_mapping: Option<&'a Arc>, + #[serde(skip_serializing_if = "Option::is_none")] + unified_partition_type: Option<&'a Arc>, + case_sensitive: bool, + #[serde(skip_serializing_if = "Option::is_none")] + key_metadata: Option<&'a [u8]>, + } + + fn partition_type( + partition_spec: Option<&PartitionSpec>, + schema: &crate::spec::Schema, + ) -> Result { + let partition_type = match partition_spec { + Some(partition_spec) => partition_spec.partition_type(schema)?, + None => PartitionSpec::unpartition_spec().partition_type(schema)?, + }; + Ok(Type::Struct(partition_type)) + } + + impl<'a> TryFrom<&'a FileScanTask> for FileScanTaskRefSerde<'a> { + type Error = Error; + + fn try_from(value: &'a FileScanTask) -> Result { + let partition = value + .partition + .as_ref() + .map(|partition| { + let partition_type = + partition_type(value.partition_spec.as_deref(), &value.schema)?; + RawLiteral::try_from(Literal::Struct(partition.clone()), &partition_type) + }) + .transpose()?; + + Ok(Self { + file_size_in_bytes: value.file_size_in_bytes, + start: value.start, + length: value.length, + record_count: value.record_count, + first_row_id: value.first_row_id, + data_sequence_number: value.data_sequence_number, + data_file_path: &value.data_file_path, + data_file_format: value.data_file_format, + schema: &value.schema, + project_field_ids: &value.project_field_ids, + predicate: value.predicate.as_ref(), + deletes: &value.deletes, + partition, + partition_spec: value.partition_spec.as_ref(), + name_mapping: value.name_mapping.as_ref(), + unified_partition_type: value.unified_partition_type.as_ref(), + case_sensitive: value.case_sensitive, + key_metadata: value.key_metadata.as_deref(), + }) + } + } + + impl Serialize for FileScanTask { + fn serialize(&self, serializer: S) -> std::result::Result + where S: serde::Serializer { + FileScanTaskRefSerde::try_from(self) + .map_err(serde::ser::Error::custom)? + .serialize(serializer) + } + } + + impl TryFrom for FileScanTask { + type Error = Error; + + fn try_from(value: FileScanTaskSerde) -> Result { + let partition = value + .partition + .map(|partition| { + let partition_type = + partition_type(value.partition_spec.as_deref(), &value.schema)?; + match partition.try_into(&partition_type)? { + Some(Literal::Struct(partition)) => Ok(partition), + _ => Err(Error::new( + ErrorKind::DataInvalid, + "FileScanTask partition must be a struct", + )), + } + }) + .transpose()?; + + Self::builder() + .with_file_size_in_bytes(value.file_size_in_bytes) + .with_start(value.start) + .with_length(value.length) + .with_record_count(value.record_count) + .with_first_row_id(value.first_row_id) + .with_data_sequence_number(value.data_sequence_number) + .with_data_file_path(value.data_file_path) + .with_data_file_format(value.data_file_format) + .with_schema(value.schema) + .with_project_field_ids(value.project_field_ids) + .with_predicate(value.predicate) + .with_deletes(value.deletes) + .with_partition(partition) + .with_partition_spec(value.partition_spec) + .with_name_mapping(value.name_mapping) + .with_unified_partition_type(value.unified_partition_type) + .with_case_sensitive(value.case_sensitive) + .with_key_metadata(value.key_metadata) + .build() + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/iceberg/src/spec/values/serde.rs b/crates/iceberg/src/spec/values/serde.rs index 10a815aa45..8a00a79dee 100644 --- a/crates/iceberg/src/spec/values/serde.rs +++ b/crates/iceberg/src/spec/values/serde.rs @@ -418,6 +418,12 @@ pub(crate) mod _serde { Type::Primitive(PrimitiveType::Timestamptz) => { Ok(Some(Literal::timestamptz(v))) } + Type::Primitive(PrimitiveType::TimestampNs) => { + Ok(Some(Literal::timestamp_nano(v))) + } + Type::Primitive(PrimitiveType::TimestamptzNs) => { + Ok(Some(Literal::timestamptz_nano(v))) + } _ => Err(invalid_err("long")), }, RawLiteralEnum::Float(v) => match ty { diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index 8c72332f13..1d716a48da 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -45,6 +45,17 @@ fn check_json_serde(json: &str, expected_literal: Literal, expected_type: &Type) assert_eq!(parsed_json_value, raw_json_value); } +fn check_raw_literal_json_serde(expected_literal: Literal, expected_type: &Type) { + let raw_literal = RawLiteral::try_from(expected_literal.clone(), expected_type).unwrap(); + let serialized = serde_json::to_string(&raw_literal).unwrap(); + let deserialized: RawLiteral = serde_json::from_str(&serialized).unwrap(); + + assert_eq!( + deserialized.try_into(expected_type).unwrap(), + Some(expected_literal) + ); +} + fn check_avro_bytes_serde(input: Vec, expected_datum: Datum, expected_type: &PrimitiveType) { let raw_schema = r#""bytes""#; let schema = apache_avro::Schema::parse_str(raw_schema).unwrap(); @@ -235,6 +246,18 @@ fn json_timestamptz_ns() { ); } +#[test] +fn raw_literal_json_serde_nanosecond_timestamps() { + check_raw_literal_json_serde( + Literal::timestamp_nano(1510871468123456789), + &Primitive(PrimitiveType::TimestampNs), + ); + check_raw_literal_json_serde( + Literal::timestamptz_nano(1510871468123456789), + &Primitive(PrimitiveType::TimestamptzNs), + ); +} + #[test] fn json_timestamptz_ns_rejects_non_utc_offset() { // Per the spec, timestamptz_ns single-value serialization must use offset "+00:00"; Java's From f347ad29cc08d18ef73ebf31231d224d0d2f872f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:37:33 +0800 Subject: [PATCH 34/74] chore(deps): Bump the codeql-action group with 2 updates (#3153) Bumps the codeql-action group with 2 updates: [github/codeql-action/init](https://github.com/github/codeql-action) and [github/codeql-action/analyze](https://github.com/github/codeql-action). Updates `github/codeql-action/init` from 4.37.8 to 4.37.9 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28...cdf488f595d80d6e07e03d4674febd5ab45fa938) Updates `github/codeql-action/analyze` from 4.37.8 to 4.37.9 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28...cdf488f595d80d6e07e03d4674febd5ab45fa938) --- updated-dependencies: - dependency-name: github/codeql-action/init dependency-version: 4.37.9 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action - dependency-name: github/codeql-action/analyze dependency-version: 4.37.9 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9d9aba21d3..7cea64aea3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -46,11 +46,11 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: category: "/language:actions" From bf80f2336293c0350b8dce9a97b2fafff1df681c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:48:42 +0800 Subject: [PATCH 35/74] chore(deps): Bump reqwest from 0.12.28 to 0.13.4 (#3152) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.28 to 0.13.4. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.12.28...v0.13.4) --- updated-dependencies: - dependency-name: reqwest dependency-version: 0.13.4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bab0d84020..010f8985eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3661,7 +3661,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -5882,7 +5882,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -5921,7 +5921,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] From 7c8345b432083482f2b0aaea3ab15460c7217152 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:49:39 +0800 Subject: [PATCH 36/74] chore(deps-dev): Bump maturin from 1.14.1 to 1.15.0 in /bindings/python (#3151) Bumps [maturin](https://github.com/pyo3/maturin) from 1.14.1 to 1.15.0. - [Release notes](https://github.com/pyo3/maturin/releases) - [Changelog](https://github.com/PyO3/maturin/blob/main/Changelog.md) - [Commits](https://github.com/pyo3/maturin/compare/v1.14.1...v1.15.0) --- updated-dependencies: - dependency-name: maturin dependency-version: 1.15.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bindings/python/uv.lock | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock index 175404bad4..0f45de4b99 100644 --- a/bindings/python/uv.lock +++ b/bindings/python/uv.lock @@ -264,26 +264,26 @@ wheels = [ [[package]] name = "maturin" -version = "1.14.1" +version = "1.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tomli", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/b3/addd877f871fb1860d46d3a4f206ecb10b946c85846805e6367631926fd3/maturin-1.14.1.tar.gz", hash = "sha256:9d6577a62cd08e0ceba7a0db06fb098e0c9b1b3429bad747a4f3a18215a1b3df", size = 369637, upload-time = "2026-06-19T05:19:49.774Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/f0/97c5a5bd9c71653a066c0976a484eaaae50b9369557838a4176b7b0bdaa5/maturin-1.14.1-py3-none-linux_armv6l.whl", hash = "sha256:522292398945442cdafa9daeb2271b2340fbde57027b818f923f88eab04174f8", size = 10207496, upload-time = "2026-06-19T05:19:09.321Z" }, - { url = "https://files.pythonhosted.org/packages/fe/83/294bca639b0e052f1e2f65199b3db258780c7d4e31408b934c9c974a1379/maturin-1.14.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:ffe5ad71f21d1e6603c4dd75f7fee34adf5ed5ebcebb692886549888ebb329ed", size = 19680113, upload-time = "2026-06-19T05:19:13.43Z" }, - { url = "https://files.pythonhosted.org/packages/43/b6/79c881410a3b1c187f7eb3d407aecae646c6a4433d630d72200359015e83/maturin-1.14.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f3306078070c1508fd715b9116070cbcaff5959024272a9f1e6f5cb29768b86c", size = 10169205, upload-time = "2026-06-19T05:19:16.615Z" }, - { url = "https://files.pythonhosted.org/packages/93/9d/44b6f26dcb7f7a04c5501ac2dbb6ca1490150682baa525ca5860504f9eab/maturin-1.14.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:cd457cd88961156e26379e1155bd287cc0ec1c8b2f1582b0660fb31b87c8842d", size = 10188098, upload-time = "2026-06-19T05:19:19.736Z" }, - { url = "https://files.pythonhosted.org/packages/1a/bd/9c0d5d6983905ce2c9edaa073a7e89355a9cf7f396988e05d32f1c37785d/maturin-1.14.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:dfc54ae32e6fcb18302193ab9a30b0b25eefffba994ae13238974805533ef75e", size = 10627576, upload-time = "2026-06-19T05:19:22.713Z" }, - { url = "https://files.pythonhosted.org/packages/e5/33/b096412bd6a7cb399652b260666f901adf88a687181a6dbd6a3f89f0a94e/maturin-1.14.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:a131d912b5267e640bc96d70f4914e10590aed64082ec9abacba7cea52004224", size = 10085181, upload-time = "2026-06-19T05:19:25.69Z" }, - { url = "https://files.pythonhosted.org/packages/56/8d/08c3bf469c38a23c9e6c877e338193001eb604d010fedc08341974e38528/maturin-1.14.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:be18fc568fb76884c0205456336892a75105ec398e6b667cd777c6268bd06d69", size = 10026363, upload-time = "2026-06-19T05:19:28.904Z" }, - { url = "https://files.pythonhosted.org/packages/3a/a4/c4d1a92839f8745ab4aab988a7db884a79d6d710bd3b286fcf9316dece1a/maturin-1.14.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:994a0c8ba3ad8a92b3a9ee1b02645d200d610216b15cff5102b0fe65e8e08666", size = 13321347, upload-time = "2026-06-19T05:19:32.411Z" }, - { url = "https://files.pythonhosted.org/packages/b3/fa/170f04624d03fd07d2a8b1b67de83a127af93aef9eaa425839553347297b/maturin-1.14.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be80866363e605d137991b491a741a84cde9ae350183c4c85f49690ca9aaaa65", size = 10877609, upload-time = "2026-06-19T05:19:35.448Z" }, - { url = "https://files.pythonhosted.org/packages/61/ad/1ae2e1d0ded282bf2c55ac13f0811d87deb425e200ae64a15785675dede9/maturin-1.14.1-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:5282dffd4b539d2be245f4e5b1a5ab6bc1033b58f4a4872f5833f9d43c954aa4", size = 10417316, upload-time = "2026-06-19T05:19:38.28Z" }, - { url = "https://files.pythonhosted.org/packages/fb/27/bf677183920718da49cd7982d6a3ffc440aad8919329f571d189f81b7bdf/maturin-1.14.1-py3-none-win32.whl", hash = "sha256:1a04de0a20188f95c721b5702eed18140bdcccb28c386797093eca3f62f4d4e0", size = 8931293, upload-time = "2026-06-19T05:19:41.183Z" }, - { url = "https://files.pythonhosted.org/packages/63/4b/585adeb9167b08d3cdff0032a938b0e72655c92003df4f52c3f696a1bcc2/maturin-1.14.1-py3-none-win_amd64.whl", hash = "sha256:3c9f94640ecc4895e94abaf834a0684430032c865b2748a36c12461fd9252fdd", size = 10314067, upload-time = "2026-06-19T05:19:44.389Z" }, - { url = "https://files.pythonhosted.org/packages/51/d4/dac8c0720ae246be1700afb6fbdbbea20fe35b13f6570b2f70faa005df77/maturin-1.14.1-py3-none-win_arm64.whl", hash = "sha256:15cea8fcb3ba47dd636f50092bb34baea8b04ac777392f23e6bf8a9a61efb894", size = 9718943, upload-time = "2026-06-19T05:19:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" }, + { url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" }, + { url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" }, + { url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" }, + { url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" }, + { url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" }, + { url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" }, ] [[package]] From c0bff81dfcf8146c0208b2fd50f1a60fbf7a2481 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 7 Sep 2026 00:02:59 -0700 Subject: [PATCH 37/74] perf(arrow): move partition key into group map instead of cloning per row (#3159) RecordBatchPartitionSplitter::split groups a batch's rows by partition value into a HashMap. It iterated the partition Structs by reference and cloned each one into the entry key, allocating a fresh Struct (and, for string partitions, copying the string) for every row even though partition_structs is never used after the loop. Consume partition_structs with into_iter() and move each Struct into the entry, dropping the per-row clone. Behavior is unchanged: grouping is by key equality, unaffected by whether the key is moved or cloned. On a 100k-row string-partitioned batch, split() drops from ~430 to ~396 ns/row (~8%); numeric partitions see a smaller win. --- crates/iceberg/src/arrow/record_batch_partition_splitter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs index 7b83621f2d..c773947c7b 100644 --- a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs +++ b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs @@ -179,10 +179,10 @@ impl RecordBatchPartitionSplitter { // Group the batch by row value. let mut group_ids = HashMap::new(); partition_structs - .iter() + .into_iter() .enumerate() .for_each(|(row_id, row)| { - group_ids.entry(row.clone()).or_insert(vec![]).push(row_id); + group_ids.entry(row).or_insert(vec![]).push(row_id); }); // Partition the batch with same partition partition_values From 1deceb1380894537c49fbb473b540c6ead5552ac Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:33:50 +0800 Subject: [PATCH 38/74] chore(deps): bump zizmor-action to 0.6.3 (#3161) --- .github/workflows/asf-allowlist-check.yml | 5 +++++ .github/workflows/audit.yml | 2 +- .github/workflows/bindings_python_ci.yml | 2 +- .github/workflows/ci.yml | 18 +++++++++--------- .github/workflows/public-api.yml | 2 +- .github/workflows/publish.yml | 4 ++-- .github/workflows/release_python.yml | 8 ++++---- .github/workflows/release_python_nightly.yml | 8 ++++---- .github/workflows/zizmor.yml | 2 +- 9 files changed, 28 insertions(+), 23 deletions(-) diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml index 3844789995..5380bcfd17 100644 --- a/.github/workflows/asf-allowlist-check.yml +++ b/.github/workflows/asf-allowlist-check.yml @@ -39,4 +39,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + # The allowlist checker recognizes ./, but not $/, as a local action prefix. + # Normalize its disposable checkout while preserving $/ in source workflows. + - name: Normalize self-repository references for the allowlist checker + run: | + find .github -type f -name '*.yml' -exec sed -i 's#uses: \$/#uses: ./#g' {} + - uses: apache/infrastructure-actions/allowlist-check@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # allowlist-check/v1.0.0 diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 02f00f67f8..c53d4af475 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -53,7 +53,7 @@ jobs: with: persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: rust-version: stable - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index a747c98842..375521d0f2 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -85,7 +85,7 @@ jobs: # Linux maturin builds may run inside manylinux Docker, so host Rust caches only help Windows/macOS. - name: Setup Rust toolchain if: runner.os != 'Linux' - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cache Rust artifacts if: runner.os != 'Linux' uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15e56b2ecc..dc380ab18c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Install lint tools # Keep versions in sync with the local install targets in Makefile. @@ -93,7 +93,7 @@ jobs: persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cargo clippy run: make check-clippy @@ -117,7 +117,7 @@ jobs: persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cache Rust artifacts uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -138,7 +138,7 @@ jobs: persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cache Rust artifacts uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -173,7 +173,7 @@ jobs: persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cache Rust artifacts uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -202,7 +202,7 @@ jobs: persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cache Rust artifacts uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -249,12 +249,12 @@ jobs: persist-credentials: false - name: Get MSRV id: get-msrv - uses: ./.github/actions/get-msrv + uses: $/.github/actions/get-msrv - name: Setup MSRV Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: rust-version: ${{ steps.get-msrv.outputs.msrv }} - name: Setup Nightly Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Check MSRV run: make check-msrv diff --git a/.github/workflows/public-api.yml b/.github/workflows/public-api.yml index eec825e242..a3af719a40 100644 --- a/.github/workflows/public-api.yml +++ b/.github/workflows/public-api.yml @@ -41,7 +41,7 @@ jobs: persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cache Rust artifacts uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index baeac2d05d..863222a2d4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,10 +43,10 @@ jobs: - name: Get MSRV id: get-msrv - uses: ./.github/actions/get-msrv + uses: $/.github/actions/get-msrv - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: rust-version: ${{ steps.get-msrv.outputs.msrv }} diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index e41cf6d7a7..758bdff21b 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -120,7 +120,7 @@ jobs: # Convert semver RC (0.9.0-rc.1) to PEP 440 (0.9.0rc1) echo "pep440=$(echo "$CARGO_VERSION" | sed 's/-rc\.\([0-9]*\)/rc\1/')" >> $GITHUB_OUTPUT - - uses: ./.github/actions/overwrite-package-version + - uses: $/.github/actions/overwrite-package-version if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} with: version: ${{ steps.version.outputs.pep440 }} @@ -167,7 +167,7 @@ jobs: # Convert semver RC (0.9.0-rc.1) to PEP 440 (0.9.0rc1) echo "pep440=$(echo "$CARGO_VERSION" | sed 's/-rc\.\([0-9]*\)/rc\1/')" >> $GITHUB_OUTPUT - - uses: ./.github/actions/overwrite-package-version + - uses: $/.github/actions/overwrite-package-version if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} with: version: ${{ steps.version.outputs.pep440 }} @@ -177,10 +177,10 @@ jobs: python-version: 3.12 - name: Get MSRV id: get-msrv - uses: ./.github/actions/get-msrv + uses: $/.github/actions/get-msrv - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: rust-version: ${{ steps.get-msrv.outputs.msrv }} - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index bd6b778499..ab2b2ff9c1 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -53,7 +53,7 @@ jobs: CURRENT=$(cargo metadata --format-version 1 --no-deps -q | jq -r '.packages[0].version') echo "pep440=${CURRENT}.dev${TIMESTAMP}" >> $GITHUB_OUTPUT - - uses: ./.github/actions/overwrite-package-version + - uses: $/.github/actions/overwrite-package-version with: version: ${{ steps.version.outputs.pep440 }} @@ -99,7 +99,7 @@ jobs: CURRENT=$(cargo metadata --format-version 1 --no-deps -q | jq -r '.packages[0].version') echo "pep440=${CURRENT}.dev${TIMESTAMP}" >> $GITHUB_OUTPUT - - uses: ./.github/actions/overwrite-package-version + - uses: $/.github/actions/overwrite-package-version with: version: ${{ steps.version.outputs.pep440 }} @@ -109,10 +109,10 @@ jobs: - name: Get MSRV id: get-msrv - uses: ./.github/actions/get-msrv + uses: $/.github/actions/get-msrv - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: rust-version: ${{ steps.get-msrv.outputs.msrv }} diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index eea0bd0983..cd4865d9e5 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -39,6 +39,6 @@ jobs: persist-credentials: false - name: Run zizmor 🌈 - uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + uses: zizmorcore/zizmor-action@70fb788f84895a7701f5643d103d587e460b5c99 # v0.6.3 with: advanced-security: false From 28ede505ebc3a274d4840624e5c85eae480a77ae Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 7 Sep 2026 01:52:28 -0700 Subject: [PATCH 39/74] chore(deps): remove dead datafusion-ffi, unused workspace deps, and stale audit ignores (#3149) Follow-up cleanup after the DataFusion TableProvider removal (#3143). - Remove `datafusion-ffi` from workspace deps; nothing inherits it since the Python binding no longer exports a TableProvider. - Remove `aes` and `thrift` workspace deps; no crate inherits them (encryption uses the AES types re-exported by `aes-gcm`). - Drop audit ignores whose advisory no longer matches any locked crate: RUSTSEC-2024-0014 (generational-arena, gone with datafusion-ffi), RUSTSEC-2024-0388 (derivative), RUSTSEC-2025-0134 (rustls-pemfile), RUSTSEC-2026-0097 (rand, all locked versions patched), RUSTSEC-2026-0176/0177 (pyo3, lock is on 0.29.1). - Drop the unmatched MPL-2.0 license exception for generational-arena. Cargo.lock is unchanged. Verified with `cargo check --workspace --all-targets --all-features`, `cargo audit`, and `cargo deny check licenses`. Co-authored-by: Claude Fable 5.1 --- .cargo/audit.toml | 27 --------------------------- Cargo.toml | 3 --- deny.toml | 1 - 3 files changed, 31 deletions(-) diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 7e31efcb19..176d43abf6 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -21,37 +21,10 @@ ignore = [ # Marvin Attack: potential key recovery through timing sidechannels # Issues: https://github.com/apache/iceberg-rust/issues/221 "RUSTSEC-2023-0071", - # `derivative` is unmaintained; consider using an alternative - # - # Introduced by hive_metastore, tracked at https://github.com/cloudwego/pilota/issues/293 - "RUSTSEC-2024-0388", # `paste` is unmaintained; consider using an alternative # # Introduced by hive_metastore, tracked at https://github.com/cloudwego/pilota/issues/293 "RUSTSEC-2024-0436", - # `generational-arena` is unmaintained; consider using an alternative - # - # Introduced by datafusion-ffi through abi_stable/async-ffi until upstream - # provides a replacement path. - "RUSTSEC-2024-0014", - # `rustls-pemfile` is unmaintained - # - # Introduced by object_store, see https://github.com/apache/arrow-rs-object-store/issues/564 - "RUSTSEC-2025-0134", - # `rand` unsoundness with custom logger using `rand::rng()` - # - # Direct dependency upgraded to 0.9.3+. Transitive rand 0.8.5 remains - # from reqsign/sqllogictest/rustc-hash — no 0.8.x patch exists. - "RUSTSEC-2026-0097", - # pyo3 < 0.29: out-of-bounds read in PyList/PyTuple `nth`/`nth_back`, and - # missing `Sync` bound on `PyCFunction::new_closure` closures. - # - # Pulled only transitively through arrow's `pyarrow` feature - # (arrow-pyarrow), which still pins pyo3 ^0.28 in its latest release; no - # arrow build supports the patched pyo3 0.29 yet. Remove once arrow-pyarrow - # moves to pyo3 >=0.29. - "RUSTSEC-2026-0176", - "RUSTSEC-2026-0177", # `quick-xml` < 0.41.0 has quadratic duplicate-attribute validation and # unbounded namespace-declaration allocation (parser DoS). # diff --git a/Cargo.toml b/Cargo.toml index 8c220f489c..d1083a3dc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,6 @@ rust-version = "1.95" unused_qualifications = "deny" [workspace.dependencies] -aes = { version = "0.8", features = ["zeroize"] } aes-gcm = "0.10" anyhow = "1.0.72" apache-avro = { version = "0.21", features = ["snappy", "zstandard"] } @@ -81,7 +80,6 @@ crc32fast = "1" dashmap = "6" datafusion = "55.0.0" datafusion-cli = "55.0.0" -datafusion-ffi = "55.0.0" datafusion-sqllogictest = "55.0.0" derive_builder = "0.20" dirs = "6" @@ -148,7 +146,6 @@ stacker = "0.1.20" strum = "0.27.2" syn = "2" tempfile = "3.18" -thrift = "0.17.0" tokio = { version = "1.47", default-features = false, features = [ "sync", "rt-multi-thread", diff --git a/deny.toml b/deny.toml index 11c6dd5dc4..e8c3985e5d 100644 --- a/deny.toml +++ b/deny.toml @@ -38,6 +38,5 @@ exceptions = [ # The MPL license is allowed (binary-only): # https://www.apache.org/legal/resolved.html#category-b { allow = ["MPL-2.0"], crate = "colored" }, - { allow = ["MPL-2.0"], crate = "generational-arena" }, { allow = ["MPL-2.0"], crate = "option-ext" }, ] From 4687d265374bf5dc7bd8cd461f3bd46a23864761 Mon Sep 17 00:00:00 2001 From: Jannik Steinmann Date: Tue, 8 Sep 2026 17:00:52 +0200 Subject: [PATCH 40/74] fix: Fix CI failure due to integration test's Hive install (#3174) Install Hive directly Signed-off-by: Jannik Steinmann --- dev/hms/Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev/hms/Dockerfile b/dev/hms/Dockerfile index 65ebc3d6d5..dff5606bdc 100644 --- a/dev/hms/Dockerfile +++ b/dev/hms/Dockerfile @@ -20,10 +20,12 @@ ENV HADOOP_VERSION=3.1.0 USER root -RUN apt-get update -qq && apt-get -qq -y install curl && \ - curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -Lo /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar && \ - curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar -Lo /opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar && \ - apt-get clean && rm -rf /var/lib/apt/lists/* +ADD --chmod=644 --checksum=sha256:a18508b9348af095ea41301e439354dbd449e304ac44c6885b2b4fe78de88126 \ + https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \ + /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar +ADD --chmod=644 --checksum=sha256:faf78ac4880f56cf52791d84ec1068ce7c66acc4295d580a726104b734c01fcd \ + https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar \ + /opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar COPY core-site.xml /opt/hadoop/etc/hadoop/core-site.xml From 7874ee2c9b958b1b09c953032cc0580669823115 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Wed, 9 Sep 2026 03:23:11 -0700 Subject: [PATCH 41/74] perf(writer): borrow statistics in MinMaxColAggregator::update instead of cloning (#3171) ParquetWriter clones the parquet Statistics for every column of every row group when collecting min/max bounds, but update() only reads it. For byte-array columns (string, binary, fixed, decimal) the clone duplicates the min and max byte buffers on the heap; for numeric columns it's a cheap copy. Take &Statistics and drop the clone at the call site. Roughly 15 ns per column per row group for byte-array columns, near nothing for numeric. Behavior is unchanged; the cheap PrimitiveType clone stays because it releases the borrow of self before the &mut self updates. --- .../src/writer/file_writer/parquet_writer.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 55defaa5d7..fbf333c7bf 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -360,7 +360,7 @@ impl MinMaxColAggregator { } /// Update statistics - fn update(&mut self, field_id: i32, value: Statistics) -> Result<()> { + fn update(&mut self, field_id: i32, value: &Statistics) -> Result<()> { let Some(ty) = self .schema .field_by_id(field_id) @@ -378,7 +378,7 @@ impl MinMaxColAggregator { }; if value.min_is_exact() { - let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, &value)? else { + let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, value)? else { return Err(Error::new( ErrorKind::Unexpected, format!("Statistics {value} is not match with field type {ty}."), @@ -389,7 +389,7 @@ impl MinMaxColAggregator { } if value.max_is_exact() { - let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, &value)? else { + let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, value)? else { return Err(Error::new( ErrorKind::Unexpected, format!("Statistics {value} is not match with field type {ty}."), @@ -488,7 +488,7 @@ impl ParquetWriter { *per_col_null_val_num.entry(field_id).or_insert(0) += null_count; } - min_max_agg.update(field_id, statistics.clone())?; + min_max_agg.update(field_id, statistics)?; } } } @@ -2523,16 +2523,16 @@ mod tests { let create_statistics = |min, max| Statistics::Int32(ValueStatistics::new(min, max, None, None, false)); min_max_agg - .update(0, create_statistics(None, Some(42))) + .update(0, &create_statistics(None, Some(42))) .unwrap(); min_max_agg - .update(0, create_statistics(Some(0), Some(i32::MAX))) + .update(0, &create_statistics(Some(0), Some(i32::MAX))) .unwrap(); min_max_agg - .update(0, create_statistics(Some(i32::MIN), None)) + .update(0, &create_statistics(Some(i32::MIN), None)) .unwrap(); min_max_agg - .update(0, create_statistics(None, None)) + .update(0, &create_statistics(None, None)) .unwrap(); let (lower_bounds, upper_bounds) = min_max_agg.produce(); From 4d83bc77dc10cff851a3ef427c451ff977bc3643 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Wed, 9 Sep 2026 12:06:22 -0700 Subject: [PATCH 42/74] perf(arrow): build repeated boolean columns with BooleanBuffer (#3178) create_primitive_array_repeated built a constant boolean column with BooleanArray::from(vec![*value; n]), which allocates an n-byte Vec and then bit-packs it. Set or clear the bits directly with BooleanBuffer::new_set / new_unset instead. The sibling string and binary arms already skip the throwaway Vec via from_iter_values, so this just brings the bool arm in line. Same output, no nulls. This path materializes constant columns (partition values, defaults, metadata) on read, and boolean constants are uncommon, so it's a minor win. --- crates/iceberg/src/arrow/value.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index 13733559c8..823a4443bb 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -23,7 +23,7 @@ use arrow_array::{ LargeListArray, LargeStringArray, ListArray, MapArray, StringArray, StructArray, Time64MicrosecondArray, TimestampMicrosecondArray, TimestampNanosecondArray, new_null_array, }; -use arrow_buffer::NullBuffer; +use arrow_buffer::{BooleanBuffer, NullBuffer}; use arrow_schema::{DataType, FieldRef, TimeUnit}; use uuid::Uuid; @@ -822,7 +822,12 @@ pub(crate) fn create_primitive_array_repeated( Ok(match (data_type, prim_lit) { // --- Primitive Some arms --- (DataType::Boolean, Some(PrimitiveLiteral::Boolean(value))) => { - Arc::new(BooleanArray::from(vec![*value; num_rows])) + let buffer = if *value { + BooleanBuffer::new_set(num_rows) + } else { + BooleanBuffer::new_unset(num_rows) + }; + Arc::new(BooleanArray::new(buffer, None)) } (DataType::Int32, Some(PrimitiveLiteral::Int(value))) => { Arc::new(Int32Array::from(vec![*value; num_rows])) From d6c2eb440db270ab2036809cec2123b4032e6504 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 10 Sep 2026 01:50:34 -0700 Subject: [PATCH 43/74] chore: exclude repo-only files from the source archive (#3189) The release tarball is built with git archive and ships .asf.yaml, .devcontainer, .github, .gitignore, .gitattributes, and .idea, none of which are needed to build, test, or verify from source. Ryan raised this in the 0.2.0 vote and Kurtis called out .idea in the 0.9.1 vote. Files the verification scripts rely on (.licenserc.yaml, lint configs, dev/) stay in. Co-authored-by: Claude Fable 5.1 --- .gitattributes | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitattributes b/.gitattributes index 908bda4b5b..465b03cbd9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,10 @@ +# Paths left out of the source archive built by dev/release/create_rc.sh. +# Files needed to build, test, and verify from source (dev/, .licenserc.yaml, +# lint configs) must stay in. website export-ignore +.asf.yaml export-ignore +.devcontainer export-ignore +.gitattributes export-ignore +.github export-ignore +.gitignore export-ignore +.idea export-ignore From 4aa55ba00c3ef2a4e603f323bfed0187952fe251 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 10 Sep 2026 02:41:14 -0700 Subject: [PATCH 44/74] fix(hms): use `get_table_req` for Hive 4 metastore compatibility (#3187) fix(hms): use get_table_req for Hive 4 metastore compatibility Hive 4.0.1 removed the `get_table` thrift method, so the HMS catalog fails with `Invalid method name: 'get_table'` against any Hive 4 metastore. Switch `load_table`, `table_exists`, and `rename_table` to `get_table_req`, which exists in every Hive release since 2.3 (the IDL the `hive_metastore` crate is generated from), so older metastores keep working. Move the integration test metastore from `apache/hive:3.1.3` (Debian Bullseye, EOL) to `apache/hive:4.2.1` so CI exercises a Hive 4 server, mirroring apache/iceberg-python#3924. Co-authored-by: Claude Fable 5.1 --- crates/catalog/hms/src/catalog.rs | 32 +++++++++++++++++++++++-------- dev/hms/Dockerfile | 18 +++++++---------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/crates/catalog/hms/src/catalog.rs b/crates/catalog/hms/src/catalog.rs index 7e5a2c391c..5ea493a5f3 100644 --- a/crates/catalog/hms/src/catalog.rs +++ b/crates/catalog/hms/src/catalog.rs @@ -23,8 +23,8 @@ use std::sync::Arc; use anyhow::anyhow; use async_trait::async_trait; use hive_metastore::{ - ThriftHiveMetastoreClient, ThriftHiveMetastoreClientBuilder, - ThriftHiveMetastoreGetDatabaseException, ThriftHiveMetastoreGetTableException, + GetTableRequest, ThriftHiveMetastoreClient, ThriftHiveMetastoreClientBuilder, + ThriftHiveMetastoreGetDatabaseException, ThriftHiveMetastoreGetTableReqException, }; use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; use iceberg::io::{FileIO, FileIOBuilder, StorageFactory}; @@ -583,10 +583,15 @@ impl Catalog for HmsCatalog { let hive_table = self .client .0 - .get_table(db_name.clone().into(), table.name.clone().into()) + .get_table_req(GetTableRequest { + db_name: db_name.clone().into(), + tbl_name: table.name.clone().into(), + ..Default::default() + }) .await .map(from_thrift_exception) - .map_err(from_thrift_error)??; + .map_err(from_thrift_error)?? + .table; let metadata_location = get_metadata_location(&hive_table.parameters)?; @@ -661,12 +666,18 @@ impl Catalog for HmsCatalog { let resp = self .client .0 - .get_table(db_name.into(), table_name.into()) + .get_table_req(GetTableRequest { + db_name: db_name.into(), + tbl_name: table_name.into(), + ..Default::default() + }) .await; match resp { Ok(MaybeException::Ok(_)) => Ok(true), - Ok(MaybeException::Exception(ThriftHiveMetastoreGetTableException::O2(_))) => Ok(false), + Ok(MaybeException::Exception(ThriftHiveMetastoreGetTableReqException::O2(_))) => { + Ok(false) + } Ok(MaybeException::Exception(exception)) => Err(Error::new( ErrorKind::Unexpected, "Operation failed for hitting thrift error".to_string(), @@ -698,10 +709,15 @@ impl Catalog for HmsCatalog { let mut tbl = self .client .0 - .get_table(src_dbname.clone().into(), src_tbl_name.clone().into()) + .get_table_req(GetTableRequest { + db_name: src_dbname.clone().into(), + tbl_name: src_tbl_name.clone().into(), + ..Default::default() + }) .await .map(from_thrift_exception) - .map_err(from_thrift_error)??; + .map_err(from_thrift_error)?? + .table; tbl.db_name = Some(dest_dbname.into()); tbl.table_name = Some(dest_tbl_name.into()); diff --git a/dev/hms/Dockerfile b/dev/hms/Dockerfile index dff5606bdc..5e6cfdbfef 100644 --- a/dev/hms/Dockerfile +++ b/dev/hms/Dockerfile @@ -13,20 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM apache/hive:3.1.3 - -ENV AWSSDK_VERSION=2.20.18 -ENV HADOOP_VERSION=3.1.0 +FROM apache/hive:4.2.1 USER root -ADD --chmod=644 --checksum=sha256:a18508b9348af095ea41301e439354dbd449e304ac44c6885b2b4fe78de88126 \ - https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \ - /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar -ADD --chmod=644 --checksum=sha256:faf78ac4880f56cf52791d84ec1068ce7c66acc4295d580a726104b734c01fcd \ - https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar \ - /opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar +# Link the hadoop-aws and AWS SDK jars that the image ships into the metastore classpath +RUN ln -s /opt/hadoop/share/hadoop/tools/lib/hadoop-aws-*.jar /opt/hive/lib/ && \ + ln -s /opt/hadoop/share/hadoop/tools/lib/bundle-*.jar /opt/hive/lib/ -COPY core-site.xml /opt/hadoop/etc/hadoop/core-site.xml +# The entrypoint links this directory into the Hive config directory, over its own core-site.xml +ENV HIVE_CUSTOM_CONF_DIR=/opt/hive/custom-conf +COPY core-site.xml ${HIVE_CUSTOM_CONF_DIR}/core-site.xml USER hive From 6262dffd764ff76be38386f0880df6cf7910d980 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 10 Sep 2026 02:45:57 -0700 Subject: [PATCH 45/74] chore(dev): run REST fixture's SQLite backend in WAL mode (#3183) The REST catalog fixture stores its JdbcCatalog in a file-backed SQLite database using the default rollback journal. Under parallel test load a committing writer must wait for every reader, and the request fails with SQLITE_BUSY ("database is locked") instead of waiting. Switching the database to WAL mode lets reads proceed alongside the writer, removing the contention entirely. Co-authored-by: Claude Fable 5.1 --- dev/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 9d288580df..f086b27d5f 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -86,7 +86,7 @@ services: - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog - - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory + - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest.db?journal_mode=WAL - CATALOG_WAREHOUSE=s3://icebergdata/demo - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - CATALOG_S3_ENDPOINT=http://minio:9000 From 6f062aa3d91541584ac0c06f5b58ff3bc172fe86 Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 10 Sep 2026 16:40:51 +0100 Subject: [PATCH 46/74] fix(ci): keep .github/actions in the source archive (#3193) * fix(ci): keep .github/actions in the source archive `.github export-ignore` (#3189) hides `.github/actions` from repository archives. GitHub resolves the `uses: $/...` self-repository references introduced in #3161 by fetching such an archive, so every workflow that uses a local composite action now fails at action resolution: Can't find 'action.yml', 'action.yaml' or 'Dockerfile' for action 'apache/iceberg-rust/.github/actions/setup-builder@' This breaks CI on main and on all open pull requests. List the .github entries individually instead, so the repo-only files #3189 targeted stay out of the release tarball while `.github/actions` remains resolvable. A `-export-ignore` negation on the subdirectory does not work: git does not descend into an export-ignored directory. * comment --- .gitattributes | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 465b03cbd9..0b27994b33 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,10 +1,12 @@ # Paths left out of the source archive built by dev/release/create_rc.sh. # Files needed to build, test, and verify from source (dev/, .licenserc.yaml, # lint configs) must stay in. +# +# .github must stay in too: `uses: $/...` action references are resolved from +# the repository archive, which honors export-ignore. website export-ignore .asf.yaml export-ignore .devcontainer export-ignore .gitattributes export-ignore -.github export-ignore .gitignore export-ignore .idea export-ignore From 0aeccb8c8d4e81e6e2e916427e183607049cd4a5 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 10 Sep 2026 15:19:35 -0700 Subject: [PATCH 47/74] fix: break dev-dependency cycle that blocks publishing iceberg-property-macro (#3184) * fix: break dev-dependency cycle that blocks publishing iceberg-property-macro `iceberg` depends on `iceberg-property-macro`, and the macro crate had a versioned dev-dependency on `iceberg` via `workspace = true`. When cargo packages the macro crate for publishing it must resolve `iceberg ^0.11.0` from crates.io, which does not exist until the next step, so `cargo publish --workspace` fails with "failed to select a version". This is the same failure that broke the 0.5.0 publish and it reproduces on the v0.11.0-rc.1 tag. Make the dev-dependency path-only so cargo strips it when packaging. Mark the macro crate as internal to `iceberg` in its description and README, and add a `cargo publish --workspace --dry-run` step to CI so an unpublishable crate fails on the PR instead of after the release vote. Co-Authored-By: Claude Fable 5.1 * ci: split publish dry-run between PR CI, RC tags, and RC creation The full `cargo publish --dry-run` compiles every packaged crate and took 13 minutes per PR. PR CI now runs it with `--no-verify`, which still catches dependency cycles, unresolvable versions, and missing metadata in seconds. The compiled dry-run moves to where it matters for a release: - `create_rc.sh` runs it before creating any artifact or the RC tag, so a crate that cannot be published fails RC creation. Opt out with `--check_publish 0`. - `publish.yml` runs it on RC tag pushes, which previously did nothing, so the exact tag is also checked on the MSRV toolchain the real publish uses. Co-Authored-By: Claude Fable 5.1 * docs: drop RC tag workflow verification note from release guide The RC creation script now blocks on the publish dry-run, so the workflow run on the tag is a secondary check that does not need a manual step. Co-Authored-By: Claude Fable 5.1 * docs: note requirements for create_rc.sh --check_publish Co-Authored-By: Claude Fable 5.1 * Apply suggestion from @dannycjones Co-authored-by: Daniel Carl Jones --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Daniel Carl Jones --- .github/workflows/ci.yml | 6 ++++++ .github/workflows/publish.yml | 7 +++++++ crates/property-macro/Cargo.toml | 6 ++++-- crates/property-macro/README.md | 5 +++++ dev/release/README.md | 6 +++++- dev/release/create_rc.sh | 24 ++++++++++++++++++++++++ website/src/release.md | 6 +++++- 7 files changed, 56 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc380ab18c..856026bd1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,12 @@ jobs: cargo check -p "$pkg" --all-targets || exit 1 done + # Dry-runs the publish without compiling, which catches dependency cycles between + # workspace crates, versions that do not resolve on crates.io, and missing metadata. + # publish.yml runs the full dry-run with compilation on release candidate tags. + - name: Check each crate can be published + run: cargo publish --workspace --all-features --dry-run --no-verify + # Verifies the core iceberg crate compiles without default features, ensuring # optional functionality is properly gated behind feature flags. build_with_no_default_features: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 863222a2d4..65ce48314e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -50,6 +50,13 @@ jobs: with: rust-version: ${{ steps.get-msrv.outputs.msrv }} + # Packages and builds every crate as it would be published, without uploading, + # so publish failures are caught during the release candidate process. + - name: Dry-run publish for release candidates + if: ${{ startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-') }} + run: cargo publish --workspace --all-features --dry-run + shell: bash + - uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 id: auth diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml index cc2e4edd6b..d48872d490 100644 --- a/crates/property-macro/Cargo.toml +++ b/crates/property-macro/Cargo.toml @@ -28,7 +28,7 @@ license = { workspace = true } repository = { workspace = true } categories = ["database"] -description = "Property derive macro for Apache Iceberg Rust" +description = "Apache Iceberg Rust property macros, internal to the iceberg crate" keywords = ["iceberg"] [lib] @@ -40,7 +40,9 @@ quote = { workspace = true } syn = { workspace = true, features = ["full"] } [dev-dependencies] -iceberg = { workspace = true } +# Path-only so cargo strips it when packaging; a versioned dev-dependency would +# create a publish cycle, since `iceberg` depends on this crate. +iceberg = { path = "../iceberg" } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } trybuild = { workspace = true } diff --git a/crates/property-macro/README.md b/crates/property-macro/README.md index ea746a4973..1bcb124263 100644 --- a/crates/property-macro/README.md +++ b/crates/property-macro/README.md @@ -19,6 +19,11 @@ # Iceberg property macros +This crate contains the property macros used internally by +[`iceberg`](https://crates.io/crates/iceberg). It is published only because +`iceberg` depends on it and has no API stability guarantees, so depend on +`iceberg` rather than on this crate. + ## `Properties` derive macro `#[derive(Properties)]` parses an owned typed struct from a flat diff --git a/dev/release/README.md b/dev/release/README.md index 48ba6286de..5c639cb007 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -42,7 +42,7 @@ Common options: dev/release/create_rc.sh 0.9.1 2 --release_ref dev/release/create_rc.sh 0.9.1 2 --create_rc_tag 0 --sign 0 dev/release/create_rc.sh 0.9.1 2 --upload_svn 1 -dev/release/create_rc.sh 0.9.1 2 --check_headers 0 --check_deps 0 +dev/release/create_rc.sh 0.9.1 2 --check_headers 0 --check_deps 0 --check_publish 0 ``` Defaults: @@ -52,6 +52,7 @@ Defaults: - `--create_rc_tag 1`: create the signed annotated RC tag as the final release step. - `--check_headers 1`: check Apache license headers against the source archive. - `--check_deps 1`: run dependency license checks before artifact creation. +- `--check_publish 1`: dry-run publishing every crate to crates.io before artifact creation. - `--sign 1`: create and verify the detached GPG signature. - `--upload_svn 0`: upload RC artifacts to the ASF dev dist SVN repository. - `--svn_dist_url https://dist.apache.org/repos/dist/dev/iceberg`: SVN directory URL where the RC artifact directory will be uploaded. @@ -64,6 +65,9 @@ Defaults: cargo install --locked cargo-deny ``` +`--check_publish 1` runs `cargo publish --workspace --dry-run`, which packages and compiles every crate and needs network access to crates.io. +Pass `--check_publish 0` to skip it when offline or when iterating on the script. + ## Verify an RC ```shell diff --git a/dev/release/create_rc.sh b/dev/release/create_rc.sh index 8f193c5ce0..940d44f080 100755 --- a/dev/release/create_rc.sh +++ b/dev/release/create_rc.sh @@ -69,6 +69,10 @@ Options: Whether to run the dependency license check before creating artifacts. Default: 1 + --check_publish <0|1> + Whether to dry-run publishing every crate to crates.io before creating artifacts. + Default: 1 + --sign <0|1> Whether to create and verify the detached GPG signature for the source archive. Default: 1 @@ -227,6 +231,12 @@ parse_args() { CHECK_DEPS="$2" shift 2 ;; + --check_publish | --check-publish) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + CHECK_PUBLISH="$2" + shift 2 + ;; --sign) require_option_value "$1" "${2:-}" validate_bool_option "$1" "$2" @@ -315,6 +325,13 @@ check_dependency_licenses() { ) } +# Packages and builds every crate as `cargo publish` would, without uploading, +# so crates that cannot be published fail RC creation. +check_crates_publishable() { + require_command cargo + cargo publish --workspace --all-features --dry-run --manifest-path "${REPO_ROOT}/Cargo.toml" +} + prepare_output_directory() { rm -rf "${RC_DIR}" mkdir -p "${RC_DIR}" @@ -500,6 +517,7 @@ RELEASE_DIST_DIR="dist" CREATE_RC_TAG="1" CHECK_HEADERS="1" CHECK_DEPS="1" +CHECK_PUBLISH="1" SIGN_ARCHIVE="1" UPLOAD_SVN="0" SVN_DIST_URL="https://dist.apache.org/repos/dist/dev/iceberg" @@ -523,6 +541,12 @@ else skip_step "Check dependency licenses" fi +if enabled "${CHECK_PUBLISH}"; then + run_step "Check crates can be published" check_crates_publishable +else + skip_step "Check crates can be published" +fi + run_step "Prepare output directory ${RC_DIR}" prepare_output_directory run_step "Create source archive ${ARCHIVE_FILE_NAME}" create_source_archive diff --git a/website/src/release.md b/website/src/release.md index 2a4f4aea87..5e207bc413 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -202,6 +202,7 @@ Useful options include: - `--create_rc_tag 1`: create the signed annotated RC tag as the final release step. - `--check_headers 1`: check Apache license headers against the source archive. - `--check_deps 1`: run dependency license checks before artifact creation. +- `--check_publish 1`: dry-run publishing every crate to crates.io before artifact creation. - `--sign 1`: create and verify the detached GPG signature. - `--upload_svn 0`: upload RC artifacts to the ASF dev dist SVN repository. - `--svn_dist_url https://dist.apache.org/repos/dist/dev/iceberg`: SVN directory URL where the RC artifact directory will be uploaded. @@ -214,7 +215,10 @@ This script creates: - SHA-512 checksum: `apache-iceberg-rust-${iceberg_version}.tar.gz.sha512` - Signed annotated RC tag: `v${iceberg_version}-rc.${rc}` -The script checks license headers against the generated source archive, not the live Git worktree. If enabled, SVN upload runs after local artifact verification and before RC tag creation. The script creates the signed RC tag as the final release step, then prints a draft VOTE email for `dev@iceberg.apache.org`. +The script runs a number of verifications. +It performs a dry-run packaging step and builds every crate as the final release would, so a crate that cannot be published fails RC creation. +It also checks license headers against the generated source archive. +If enabled, SVN upload runs after local artifact verification and before RC tag creation. The script creates the signed RC tag as the final release step, then prints a draft VOTE email for `dev@iceberg.apache.org`. To upload artifacts to ASF dev dist as part of RC creation, pass: From 44ceed3015a97fc18375dbf09758b42bc264db0e Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 11 Sep 2026 02:20:55 -0700 Subject: [PATCH 48/74] test(arrow): cover repeated boolean array path (#3201) Follow-up to #3178 (https://github.com/apache/iceberg-rust/pull/3178), which optimized the boolean arm of create_primitive_array_repeated but was not exercised by existing tests. Add a test over true / false / num_rows == 0 asserting len, null_count, and element values. --- crates/iceberg/src/arrow/value.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index 823a4443bb..ff28052fce 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -1985,6 +1985,34 @@ mod test { assert!((0..num_rows).all(|i| fixed.value(i) == bytes.as_slice())); } + #[test] + fn test_create_boolean_array_repeated() { + let num_rows = 4; + + for value in [true, false] { + let array = create_primitive_array_repeated( + &DataType::Boolean, + Some(&PrimitiveLiteral::Boolean(value)), + num_rows, + ) + .unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); + assert_eq!(array.len(), num_rows); + assert_eq!(array.null_count(), 0); + assert!((0..num_rows).all(|i| array.value(i) == value)); + } + + // num_rows == 0 must produce an empty (not one-element) array. + let empty = create_primitive_array_repeated( + &DataType::Boolean, + Some(&PrimitiveLiteral::Boolean(true)), + 0, + ) + .unwrap(); + assert_eq!(empty.len(), 0); + assert_eq!(empty.null_count(), 0); + } + #[test] fn test_create_string_array_repeated_empty() { // num_rows == 0 must produce an empty (not one-element) array. From 8ac8b1c2ab3bd07e284cf096de003033ec489e4c Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 11 Sep 2026 05:51:19 -0700 Subject: [PATCH 49/74] perf(arrow): build constant filter masks with BooleanBuffer instead of Vec (#3177) * perf(arrow): build constant filter masks with BooleanBuffer instead of Vec The predicate converter builds all-true and all-false boolean masks with BooleanArray::from(vec![b; n]), which allocates an n-byte Vec and then bit-packs it into the array. BooleanBuffer::new_set/new_unset write the packed bits directly, skipping the throwaway Vec and the pack pass. Covers the always-true/false predicates and the initial accumulators for is_in / not_in, built once per batch. Output is identical (all-set/all-unset, no nulls). Isolated construction is ~200x faster at 8192 rows; the real per-batch win is smaller since this is a fraction of filter evaluation. * PR feedback --- .../src/arrow/reader/predicate_visitor.rs | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 272de49390..2b05e3ebc3 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -246,18 +246,29 @@ impl PredicateConverter<'_> { /// Build an Arrow predicate that always returns true. fn build_always_true(&self) -> Result> { Ok(Box::new(|batch| { - Ok(BooleanArray::from(vec![true; batch.num_rows()])) + Ok(constant_bool_array(true, batch.num_rows())) })) } /// Build an Arrow predicate that always returns false. fn build_always_false(&self) -> Result> { Ok(Box::new(|batch| { - Ok(BooleanArray::from(vec![false; batch.num_rows()])) + Ok(constant_bool_array(false, batch.num_rows())) })) } } +/// Builds a non-null `BooleanArray` of `len` elements all set to `value`. +fn constant_bool_array(value: bool, len: usize) -> BooleanArray { + let buffer = if value { + BooleanBuffer::new_set(len) + } else { + BooleanBuffer::new_unset(len) + }; + + BooleanArray::new(buffer, None) +} + /// Gets the leaf column from the record batch for the required column index. Only /// supports top-level columns for now. fn project_column( @@ -590,7 +601,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { // update this if arrow ever adds a native is_in kernel let left = project_column(&batch, idx)?; - let mut acc = BooleanArray::from(vec![false; batch.num_rows()]); + let mut acc = constant_bool_array(false, batch.num_rows()); for literal in &literals { let literal = try_cast_literal(literal, left.data_type())?; acc = or(&acc, &eq(&left, literal.as_ref())?)? @@ -619,7 +630,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { Ok(Box::new(move |batch| { // update this if arrow ever adds a native not_in kernel let left = project_column(&batch, idx)?; - let mut acc = BooleanArray::from(vec![true; batch.num_rows()]); + let mut acc = constant_bool_array(true, batch.num_rows()); for literal in &literals { let literal = try_cast_literal(literal, left.data_type())?; acc = and(&acc, &neq(&left, literal.as_ref())?)? @@ -665,7 +676,7 @@ mod tests { use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; - use super::{CollectFieldIdVisitor, PredicateConverter}; + use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::{Bind, Predicate, Reference}; use crate::spec::{NestedField, PrimitiveType, Schema, SchemaRef, Type}; @@ -743,6 +754,21 @@ mod tests { assert_eq!(visitor.field_ids, expected); } + #[test] + fn test_constant_bool_array() { + for len in [0, 8192] { + let all_true = constant_bool_array(true, len); + assert_eq!(all_true.len(), len); + assert_eq!(all_true.null_count(), 0); + assert!(all_true.iter().all(|v| v == Some(true))); + + let all_false = constant_bool_array(false, len); + assert_eq!(all_false.len(), len); + assert_eq!(all_false.null_count(), 0); + assert!(all_false.iter().all(|v| v == Some(false))); + } + } + fn apply_predicate_to_batch( predicate: Predicate, schema: SchemaRef, From cb912c91066f4fc9bca8b5cdebc40e79aef38d9f Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 11 Sep 2026 15:30:44 -0700 Subject: [PATCH 50/74] ci: check new crates are reserved on crates.io before merge (#3185) * ci: check new crates are reserved on crates.io before merge Trusted publishing can't create a crate, and the release workflow publishes the whole workspace in one `cargo publish --workspace` run, which cargo aborts if any crate already exists at the release version. So the existing "publish the new crate by hand after the workflow fails" instructions no longer work. Add a lint step that fails if a publishable crate is missing from crates.io, and rewrite the release guide appendix around reserving the crate with a placeholder version when it is added. Co-Authored-By: Claude Fable 5.1 * docs: reserve the crate name with a standalone placeholder crate Overriding the version inside the workspace fails because iceberg depends on the new crate at the workspace version. Verified with cargo publish --dry-run. Co-Authored-By: Claude Fable 5.1 * ci: match the existing publishable-crate filter; document partial publish recovery Co-Authored-By: Claude Fable 5.1 * docs: clarify that reserving a crate needs a token with the publish-new scope Owning the existing crates does not imply the token in ~/.cargo/credentials.toml has publish-new, and crates.io reports a missing scope as a generic authentication failure. Say so, and point at where to create the token. Co-Authored-By: Claude Fable 5.1 * docs: fix stale first-publish note, empty placeholder, and owner invitations - The release steps still said the publish workflow is expected to fail on new crates and linked to a heading that no longer exists. - `cargo new --lib` generates a non-empty lib.rs; use mkdir and touch. - `cargo owner --add ` sends an invitation that must be accepted. Co-Authored-By: Claude Fable 5.1 * ci: fail the crates.io check if the crate list is empty A failed `cargo metadata` inside `for pkg in $(...)` never trips `bash -e`, so the loop ran zero times and the step passed without checking anything. Capture the list first and fail if it is empty. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- .github/workflows/ci.yml | 16 +++++++ website/src/release.md | 101 +++++++++++++++++++++++++++++---------- 2 files changed, 92 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 856026bd1a..5bb2cc178d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,22 @@ jobs: - name: Check crates package LICENSE and NOTICE run: make check-license-notice + # Trusted publishing cannot create a crate (https://crates.io/docs/trusted-publishing), + # so a new publishable crate must be reserved on crates.io before it is merged. + # See "Adding a new crate" in website/src/release.md. + - name: Check publishable crates exist on crates.io + run: | + pkgs=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.publish != []) | .name') + # A failed `cargo metadata` would otherwise leave the loop empty and the step green. + [ -n "$pkgs" ] || { echo "::error::No publishable crates found in the workspace, expected at least one"; exit 1; } + for pkg in $pkgs; do + # --registry is required, otherwise cargo resolves the local workspace member. + if ! cargo info --registry crates-io "$pkg" >/dev/null; then + echo "::error::$pkg is not on crates.io. Reserve it before merging; see 'Adding a new crate' in website/src/release.md" + exit 1 + fi + done + - name: Cargo Machete run: cargo machete diff --git a/website/src/release.md b/website/src/release.md index 5e207bc413..4b97fc6f75 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -531,9 +531,8 @@ The creation of the final release tag triggers the publish workflow for crates. Python packages are manually triggered later. Please verify that the triggered workflows for the crates succeeded. -Note, this workflow for crates is expected to fail if new crates are being published for the first time. -In this instance, a committer must manually publish the crate in order to continue. -See [publishing a crate for the first time](#publishing-a-crate-for-the-first-time) for what steps to take. +New crates are reserved on crates.io when they are added, so the workflow should not fail on a new crate. +If it does, see [if the Publish workflow fails on a missing crate](#if-the-publish-workflow-fails-on-a-missing-crate). Once all the crates are published, Python publishing should start. Python publishing is performed by a GitHub workflow, however the trigger is manual. @@ -614,45 +613,97 @@ however it is in the community's best interest to publish it soon after. ## Appendix -### Publishing a crate for the first time +### Adding a new crate -Publishing the Iceberg crates is automated using GitHub Actions, -which authenticates with crates.io using trusted publishing. +Every publishable crate must exist on crates.io before the pull request that adds it is merged. +CI enforces this with a check that fails for any publishable crate missing from crates.io. -Trusted publishing must first be configured for each crate. -If the crate has never been published, crates.io rejects attempts to publish. +The release workflow publishes crates with [trusted publishing](https://crates.io/docs/trusted-publishing), which cannot create a crate. +crates.io requires the first version of a crate to be published manually with an API token ([announcement](https://blog.rust-lang.org/2025/07/11/crates-io-development-update-2025-07/)). +Publishing the crate by hand after the release workflow fails does not work either: the workflow runs `cargo publish --workspace`, which refuses to run if any crate already exists at the release version. -When a release workflow fails due to the presence of a new crate, a committer must perform steps enumerated below. -Once complete, future versions can be published automatically using GitHub Actions. +A committer therefore reserves the crate when it is added, in three steps: publish a placeholder version, configure owners, and configure trusted publishing. -#### Initial crate publish +#### Prerequisites -Manually publish the crate using the following command. -You **must** have the source code checked out matching the pushed Git tag for the release. +Any crates.io account can publish an unclaimed name, and the first publisher becomes the crate's only owner. +Reserve the crate promptly once the pull request is open, and have a committer do it, since the reserver must hand ownership to the project. + +The committer needs: + +- A crates.io account linked to their GitHub account. +- Membership in the `apache/iceberg-private` GitHub team, with the `read:org` permission granted to crates.io, in order to add that team as an owner. +- A crates.io API token with the `publish-new` and `change-owners` scopes, used with `cargo login`. + Existing tokens usually lack `publish-new`; create one at [crates.io/settings/tokens](https://crates.io/settings/tokens). + +Committers who already own the existing Iceberg crates meet the first two. +List them with `cargo owner --list iceberg`. + +#### Step 1: Reserve the crate name + +Publish an empty placeholder crate as version `0.0.0`, so it never collides with a real release. +The placeholder is created outside the repository; overriding the version inside the workspace does not work because other workspace crates depend on the new crate at the workspace version. ```shell -cargo publish --package +mkdir -p /tmp//src +cd /tmp/ +touch src/lib.rs +cat > Cargo.toml <<'EOF' +[package] +name = "" +version = "0.0.0" +edition = "2024" +license = "Apache-2.0" +description = "Placeholder that reserves the crate name for Apache Iceberg Rust. See the repository for the real crate." +repository = "https://github.com/apache/iceberg-rust" +EOF +cargo publish --dry-run +cargo publish ``` -#### Configure crate permissions +A `403 Forbidden: authentication failed` error means the token lacks the `publish-new` scope. -After publishing succeeds, the crate must be configured to allow other committers to publish. +#### Step 2: Configure crate owners -Add the GitHub team that Apache Iceberg committers are a member of. +Add the GitHub team that owns the existing Iceberg crates. ```shell -cargo owner --add github:: +cargo owner --add github:apache:iceberg-private ``` -Additionally, add two PMC members (excluding yourself) as owners of the crate. -A GitHub team cannot manage permissions for the crate, so it is important that individuals have ownership to continue being able to manage the crate should a PMC member become inactive. -See the [Cargo documentation for `cargo owner`](https://doc.rust-lang.org/cargo/reference/publishing.html#cargo-owner) for reference. +Then add two PMC members other than yourself as individual owners. ```shell -cargo owner --add +cargo owner --add ``` -#### Configure trusted publishing +This sends an invitation; ask each invitee to accept it on crates.io. + +Individual owners are required because a team owner can publish and yank versions but cannot add or remove owners. +Without them, the crate could become unmanageable if a PMC member becomes inactive. +See the [Cargo documentation for `cargo owner`](https://doc.rust-lang.org/cargo/reference/publishing.html#cargo-owner). + +#### Step 3: Configure trusted publishing + +As a crate owner, open the crate's settings on the crates.io website and add a trusted publisher, following the [crates.io trusted publishing documentation](https://crates.io/docs/trusted-publishing). +Use an existing Iceberg crate as the reference: -Review the [crates.io trusted publishing documentation](https://crates.io/docs/trusted-publishing) for the latest instructions on how to configure it. -You should also review another already-existing crate for reference. +- Repository: `apache/iceberg-rust` +- Workflow: `publish.yml` +- Environment: `publish` + +Then re-run CI on the pull request. +The next release publishes the crate's first real version along with the rest of the workspace. + +#### If the Publish workflow fails on a missing crate + +Reserve the crate as above, using the `0.0.0` placeholder and never the release version. + +Then check the failed run's log for `Uploaded` lines to see which crates were already published, since cargo uploads crates one at a time in dependency order. + +- If nothing was uploaded, re-run the failed workflow run from the GitHub Actions UI. Re-running keeps the original tag as the workflow ref. +- If some crates were uploaded, publish the rest from a clean checkout of the release tag, excluding each crate that was already published: + +```shell +cargo publish --workspace --all-features --exclude --exclude +``` From 100804c7148b65b751d8d39ef671f7306cbaaedf Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 11 Sep 2026 15:54:12 -0700 Subject: [PATCH 51/74] ci: enable GitHub merge queue with required CI gates (#3195) Closes #3140. Mirror apache/iceberg-python#3832: define the required checks in .asf.yaml, add an ASF-managed Merge Queue ruleset (one-at-a-time squash merges on main), enable auto-merge, and make every required workflow run on merge_group without pull_request path or branch filters so its check always reports. CI and Bindings Python CI gain an always-run aggregate job (ci-required, bindings-python-ci-required) so matrix job names do not have to be enumerated as required contexts. Co-authored-by: Claude Fable 5.1 --- .asf.yaml | 40 +++++++++++++++++++++-- .github/workflows/asf-allowlist-check.yml | 1 + .github/workflows/bindings_python_ci.yml | 28 ++++++++-------- .github/workflows/ci.yml | 28 ++++++++-------- .github/workflows/codeql.yml | 2 +- .github/workflows/public-api.yml | 3 +- .github/workflows/zizmor.yml | 2 +- 7 files changed, 72 insertions(+), 32 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 1b32ea0132..974c6b512d 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -36,15 +36,51 @@ github: protected_branches: main: required_status_checks: - # strict means "Require branches to be up to date before merging". - strict: true + # Check context names (job names, not workflow files) that Merge Queue + # waits for. Every producer workflow must run on both `pull_request` + # and `merge_group`, without path filters, so each context always + # reports. `strict` is omitted: Merge Queue validates every entry + # against an up-to-date main, and ASF defaults it to false. + contexts: + # Repository policy and security checks. + - Analyze Actions + - Run zizmor 🌈 + - asf-allowlist-check + + # Rust workflow gates. + - ci-required + - check-public-api + + # Python bindings workflow gates. + - bindings-python-ci-required required_pull_request_reviews: required_approving_review_count: 1 dismiss_stale_reviews: false required_linear_history: true + rulesets: + - name: Merge Queue + target: branch + enforcement: active + conditions: + ref_name: + include: + - "~DEFAULT_BRANCH" + exclude: [] + rules: + - type: merge_queue + parameters: + check_response_timeout_minutes: 90 + grouping_strategy: ALLGREEN + max_entries_to_build: 1 + max_entries_to_merge: 1 + merge_method: SQUASH + min_entries_to_merge: 1 + min_entries_to_merge_wait_minutes: 0 pull_requests: + # allow pull requests to merge automatically once all requirements are met + allow_auto_merge: true # auto-delete head branches after being merged del_branch_on_merge: true features: diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml index 5380bcfd17..446a247cd1 100644 --- a/.github/workflows/asf-allowlist-check.yml +++ b/.github/workflows/asf-allowlist-check.yml @@ -25,6 +25,7 @@ name: "ASF Allowlist Check" on: pull_request: + merge_group: push: branches: - main diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 375521d0f2..24ba728d62 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -22,19 +22,7 @@ on: branches: - main pull_request: - paths: - - '**' # Include all files and directories in the repository by default. - - '!.github/ISSUE_TEMPLATE/**' # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation. - - '!dev/release/**' - - '!website/**' - - '!.asf.yml' - - '!.gitattributes' - - '!.gitignore' - - '!CONTRIBUTING.md' - - '!CHANGELOG.md' - - '!LICENSE' - - '!NOTICE' - - '!README.md' + merge_group: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} @@ -121,3 +109,17 @@ jobs: HF_DATASET: ${{ secrets.HF_DATASET }} run: | make test + + bindings-python-ci-required: + if: ${{ always() }} + needs: [check-python, test] + runs-on: ubuntu-slim + steps: + - name: Verify Bindings Python CI jobs succeeded + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + read -ra results <<< "$RESULTS" + for result in "${results[@]}"; do + test "$result" = "success" + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bb2cc178d..f3de07ff70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,19 +22,7 @@ on: branches: - main pull_request: - paths: - # Artifacts that must be included in releases, such as LICENSE and NOTICE, are deliberately not excluded so we can verify they will be bundled. - # Include all files and directories, then filter out files and directories that don't impact code, tests, or release artifacts. - - '**' - - '!.github/ISSUE_TEMPLATE/**' - - '!dev/release/**' - - '!website/**' - - '!.asf.yml' - - '!.gitattributes' - - '!.gitignore' - - '!CONTRIBUTING.md' - - '!CHANGELOG.md' - - '!README.md' + merge_group: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} @@ -280,3 +268,17 @@ jobs: uses: $/.github/actions/setup-builder - name: Check MSRV run: make check-msrv + + ci-required: + if: ${{ always() }} + needs: [lint, clippy, build, check_standalone, build_with_no_default_features, tests, msrv] + runs-on: ubuntu-slim + steps: + - name: Verify CI jobs succeeded + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + read -ra results <<< "$RESULTS" + for result in "${results[@]}"; do + test "$result" = "success" + done diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7cea64aea3..efd720e534 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,7 +23,7 @@ on: push: branches: [ "main" ] pull_request: - branches: [ "main" ] + merge_group: schedule: - cron: '16 4 * * 1' diff --git a/.github/workflows/public-api.yml b/.github/workflows/public-api.yml index a3af719a40..3603247594 100644 --- a/.github/workflows/public-api.yml +++ b/.github/workflows/public-api.yml @@ -22,8 +22,7 @@ on: branches: - main pull_request: - paths: - - 'crates/**' + merge_group: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index cd4865d9e5..bbc35f71f9 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -23,7 +23,7 @@ on: push: branches: ["main"] pull_request: - branches: ["**"] + merge_group: permissions: {} From 9e1670da2f131dede1e94a5d247a5add09ccd80f Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Fri, 11 Sep 2026 23:44:53 +0000 Subject: [PATCH 52/74] ci: pull MinIO images from Quay (#3202) Docker Hub no longer serves the MinIO server and client repositories, preventing integration test containers from starting. Use the corresponding Quay images with the existing release tags. Generated-by: Codex Co-authored-by: Kevin Liu --- dev/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index f086b27d5f..16c999cb9a 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -28,7 +28,7 @@ services: # MinIO - S3-compatible storage (shared by all tests) # ============================================================================= minio: - image: minio/minio:RELEASE.2025-05-24T17-08-30Z + image: quay.io/minio/minio:RELEASE.2025-05-24T17-08-30Z environment: - MINIO_ROOT_USER=admin - MINIO_ROOT_PASSWORD=password @@ -56,7 +56,7 @@ services: depends_on: minio: condition: service_healthy - image: minio/mc:RELEASE.2025-05-21T01-59-54Z + image: quay.io/minio/mc:RELEASE.2025-05-21T01-59-54Z environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password From 682ceb9afdf2b54a91bdd04e70dc66429f6797d4 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Fri, 11 Sep 2026 23:58:41 +0000 Subject: [PATCH 53/74] chore(release): Remove --release_ref from create_rc.sh, always release HEAD (#3200) Remove release_ref from create_rc.sh Co-authored-by: Kevin Liu --- dev/release/README.md | 4 ++-- dev/release/create_rc.sh | 23 ++++++++--------------- website/src/release.md | 3 ++- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 5c639cb007..24c014b0d1 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -36,10 +36,11 @@ This creates: The license header check runs against the generated source archive, not the live Git worktree. The optional SVN upload runs after local artifact verification and before RC tag creation. The signed RC tag is created as the final release step, then the script prints a draft VOTE email for `dev@iceberg.apache.org`. The script logs every step before it runs and after it succeeds. If a step fails, it prints the failed step and stops. +The script always archives and tags `HEAD`, so check out the exact commit to release before running it. + Common options: ```shell -dev/release/create_rc.sh 0.9.1 2 --release_ref dev/release/create_rc.sh 0.9.1 2 --create_rc_tag 0 --sign 0 dev/release/create_rc.sh 0.9.1 2 --upload_svn 1 dev/release/create_rc.sh 0.9.1 2 --check_headers 0 --check_deps 0 --check_publish 0 @@ -47,7 +48,6 @@ dev/release/create_rc.sh 0.9.1 2 --check_headers 0 --check_deps 0 --check_publis Defaults: -- `--release_ref HEAD`: git commit-ish to archive and tag. - `--dist_dir dist`: artifact output root. - `--create_rc_tag 1`: create the signed annotated RC tag as the final release step. - `--check_headers 1`: check Apache license headers against the source archive. diff --git a/dev/release/create_rc.sh b/dev/release/create_rc.sh index 940d44f080..54b8596fda 100755 --- a/dev/release/create_rc.sh +++ b/dev/release/create_rc.sh @@ -47,11 +47,10 @@ Arguments: Numeric release candidate round. Example: 2 creates tag v0.9.1-rc.2 and dist dir apache-iceberg-rust-0.9.1-rc2. -Options: - --release_ref - Git commit-ish to archive and tag. - Default: HEAD +The script always archives and tags HEAD, so check out the exact commit to +release before running it. +Options: --dist_dir Directory where RC artifacts are written. Relative paths are resolved from the repository root. @@ -92,7 +91,7 @@ Examples: $0 0.9.1 2 $0 0.9.1 2 --create_rc_tag 0 --sign 0 $0 0.9.1 2 --upload_svn 1 - $0 0.9.1 2 --release_ref abc123 --dist_dir /tmp/iceberg-rust-dist + $0 0.9.1 2 --dist_dir /tmp/iceberg-rust-dist USAGE } @@ -203,11 +202,6 @@ parse_args() { usage exit 0 ;; - --release_ref | --release-ref) - require_option_value "$1" "${2:-}" - RELEASE_REF="$2" - shift 2 - ;; --dist_dir | --dist-dir) require_option_value "$1" "${2:-}" RELEASE_DIST_DIR="$2" @@ -305,7 +299,7 @@ derive_release_names() { check_release_ref() { require_command git - git -C "${REPO_ROOT}" rev-parse --verify "${RELEASE_REF}^{commit}" >/dev/null + git -C "${REPO_ROOT}" rev-parse --verify "HEAD^{commit}" >/dev/null } check_rc_tag_available() { @@ -343,7 +337,7 @@ create_source_archive() { --format=tar.gz \ --output="${RC_DIR}/${ARCHIVE_FILE_NAME}" \ --prefix="${ARCHIVE_BASE_NAME}/" \ - "${RELEASE_REF}" + HEAD } check_license_headers() { @@ -434,7 +428,7 @@ upload_to_svn() { create_rc_tag() { require_command git require_gpg_secret_key - git -C "${REPO_ROOT}" tag -s "${RC_TAG}" "${RELEASE_REF}" -m "Apache Iceberg Rust ${VERSION} RC${RC}" + git -C "${REPO_ROOT}" tag -s "${RC_TAG}" HEAD -m "Apache Iceberg Rust ${VERSION} RC${RC}" } print_summary() { @@ -512,7 +506,6 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" VERSION="" RC="" -RELEASE_REF="HEAD" RELEASE_DIST_DIR="dist" CREATE_RC_TAG="1" CHECK_HEADERS="1" @@ -527,7 +520,7 @@ run_step "Parse command arguments" parse_args "$@" run_step "Validate release arguments" validate_args derive_release_names -run_step "Check release reference ${RELEASE_REF}" check_release_ref +run_step "Check release reference HEAD" check_release_ref if enabled "${CREATE_RC_TAG}"; then run_step "Check RC tag ${RC_TAG} is available" check_rc_tag_available diff --git a/website/src/release.md b/website/src/release.md index 4b97fc6f75..777c770f50 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -195,9 +195,10 @@ For example: dev/release/create_rc.sh 0.9.1 2 ``` +The script always archives and tags `HEAD`, so make sure the exact commit to release is checked out before running it. + Useful options include: -- `--release_ref HEAD`: git commit-ish to archive and tag. - `--dist_dir dist`: artifact output root. - `--create_rc_tag 1`: create the signed annotated RC tag as the final release step. - `--check_headers 1`: check Apache license headers against the source archive. From 29929f6b6207dcf2ebf8e93c634adf9b7b643264 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 11 Sep 2026 23:59:19 +0000 Subject: [PATCH 54/74] ci: avoid redundant post-merge runs (#3203) Merge queue checks already validate the final combined change before it lands, so stop rerunning required workflows on main after merge. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/asf-allowlist-check.yml | 3 --- .github/workflows/bindings_python_ci.yml | 3 --- .github/workflows/ci.yml | 3 --- .github/workflows/public-api.yml | 3 --- .github/workflows/zizmor.yml | 2 -- 5 files changed, 14 deletions(-) diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml index 446a247cd1..22fdb8f407 100644 --- a/.github/workflows/asf-allowlist-check.yml +++ b/.github/workflows/asf-allowlist-check.yml @@ -26,9 +26,6 @@ name: "ASF Allowlist Check" on: pull_request: merge_group: - push: - branches: - - main permissions: contents: read diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 24ba728d62..f804510f05 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -18,9 +18,6 @@ name: Bindings Python CI on: - push: - branches: - - main pull_request: merge_group: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3de07ff70..ef36027fdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,9 +18,6 @@ name: CI on: - push: - branches: - - main pull_request: merge_group: diff --git a/.github/workflows/public-api.yml b/.github/workflows/public-api.yml index 3603247594..c99b365b91 100644 --- a/.github/workflows/public-api.yml +++ b/.github/workflows/public-api.yml @@ -18,9 +18,6 @@ name: Public API on: - push: - branches: - - main pull_request: merge_group: diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index bbc35f71f9..5574a9f95a 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -20,8 +20,6 @@ name: GitHub Actions Security Analysis with zizmor 🌈 on: - push: - branches: ["main"] pull_request: merge_group: From be48041fc64a661eb6c39530d3a0d19d7134cf51 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 12 Sep 2026 01:45:46 +0000 Subject: [PATCH 55/74] ci: refresh Rust caches on main changes (#3205) * ci: warm shared Rust caches on main Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * ci: restrict cache writes to main Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * ci: refresh Rust caches in main workflows Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * ci: keep cache hydration change focused Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * ci: reuse prior workflow path filters Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/bindings_python_ci.yml | 16 ++++++++++++++++ .github/workflows/ci.yml | 16 ++++++++++++++++ .github/workflows/public-api.yml | 5 +++++ 3 files changed, 37 insertions(+) diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index f804510f05..84ce59fca8 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -18,6 +18,22 @@ name: Bindings Python CI on: + push: + branches: + - main + paths: + - '**' # Include all files and directories in the repository by default. + - '!.github/ISSUE_TEMPLATE/**' # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation. + - '!dev/release/**' + - '!website/**' + - '!.asf.yml' + - '!.gitattributes' + - '!.gitignore' + - '!CONTRIBUTING.md' + - '!CHANGELOG.md' + - '!LICENSE' + - '!NOTICE' + - '!README.md' pull_request: merge_group: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef36027fdc..e1c6b94d59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,22 @@ name: CI on: + push: + branches: + - main + paths: + # Artifacts that must be included in releases, such as LICENSE and NOTICE, are deliberately not excluded so we can verify they will be bundled. + # Include all files and directories, then filter out files and directories that don't impact code, tests, or release artifacts. + - '**' + - '!.github/ISSUE_TEMPLATE/**' + - '!dev/release/**' + - '!website/**' + - '!.asf.yml' + - '!.gitattributes' + - '!.gitignore' + - '!CONTRIBUTING.md' + - '!CHANGELOG.md' + - '!README.md' pull_request: merge_group: diff --git a/.github/workflows/public-api.yml b/.github/workflows/public-api.yml index c99b365b91..9b19358fd0 100644 --- a/.github/workflows/public-api.yml +++ b/.github/workflows/public-api.yml @@ -18,6 +18,11 @@ name: Public API on: + push: + branches: + - main + paths: + - 'crates/**' pull_request: merge_group: From 8ac2b81f81effe582471b209be2e455876fe8423 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 14 Sep 2026 07:41:51 +0000 Subject: [PATCH 56/74] Infra: Switch Dependabot to a monthly cadence (#3216) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/dependabot.yml | 9 +++------ website/src/release.md | 10 ++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 47d3022617..f4af99ef5c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,8 +21,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" - day: "sunday" + interval: "monthly" cooldown: default-days: 7 groups: @@ -37,8 +36,7 @@ updates: - package-ecosystem: "cargo" directory: "/" schedule: - interval: "weekly" - day: "sunday" + interval: "monthly" open-pull-requests-limit: 50 versioning-strategy: lockfile-only ignore: @@ -58,7 +56,6 @@ updates: - package-ecosystem: "uv" directory: "/bindings/python" schedule: - interval: "weekly" - day: "sunday" + interval: "monthly" cooldown: default-days: 7 diff --git a/website/src/release.md b/website/src/release.md index 777c770f50..116e175645 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -121,6 +121,16 @@ For details on how to run a release, please refer to: https://rust.iceberg.apach The following steps should be followed once the release is ready to begin. +### Update dependencies + +Dependabot runs monthly to reduce update noise, so dependencies may be out of date by the time of a release. +Before cutting a release candidate, open the repository's [Dependabot page](https://github.com/apache/iceberg-rust/network/updates) and trigger a manual check for each configured ecosystem: `cargo` and `github-actions` at `/`, and `uv` at `/bindings/python`. +Review and merge the applicable dependency update PRs before creating the release branch and regenerating the dependency lists. +For an existing release branch, backport the applicable updates to that branch before regenerating its dependency lists and cutting the release candidate. + +Only a committer with write access to the repository can trigger Dependabot manually. +Work with a committer if you do not have write access. + ### Create a minor version release branch, if it doesn't already exist A committer must ensure that a release branch exists for each minor version. From 0f2229c135c8c2693804a1e752e54f940696e90c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:44:38 +0000 Subject: [PATCH 57/74] chore(deps): Bump crate-ci/typos from 1.50.0 to 1.50.1 (#3213) Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.50.0 to 1.50.1. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/4d9c206a77c041268485162b8e2579ad7a5cb9a3...d43b6c087ac471e2ea7b8af622ff15f05c0c365b) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.50.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1c6b94d59..34cf9a7ed8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: run: cargo machete - name: Check typos - uses: crate-ci/typos@4d9c206a77c041268485162b8e2579ad7a5cb9a3 # v1.50.0 + uses: crate-ci/typos@d43b6c087ac471e2ea7b8af622ff15f05c0c365b # v1.50.1 clippy: needs: lint From 0c476ab3af9c0498a79356bf0fc54c119356f2c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:45:54 +0000 Subject: [PATCH 58/74] chore(deps): Bump reqwest from 0.12.28 to 0.13.4 (#3211) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.28 to 0.13.4. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.12.28...v0.13.4) --- updated-dependencies: - dependency-name: reqwest dependency-version: 0.13.4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 010f8985eb..bab0d84020 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3661,7 +3661,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -5882,7 +5882,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.4", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -5921,7 +5921,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.5.10", "tracing", "windows-sys 0.61.2", ] From 342a6ea7fe70ad8f475f8a9c75602f5f899742a2 Mon Sep 17 00:00:00 2001 From: blackmwk <257669749+blackmwk@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:46:07 +0000 Subject: [PATCH 59/74] refactor(sql): derive catalog configuration from properties (#3105) * refactor(sql): derive catalog configuration from properties Use the shared property derive for SQL catalog settings while preserving builder precedence, bind-style aliases, schema validation, and forwarded pool properties. Tracks [apache/iceberg-rust#3099](https://github.com/apache/iceberg-rust/issues/3099). Generated-by: Codex * refactor(sql): centralize catalog properties * fix(sql): address catalog property review --- Cargo.lock | 1 + crates/catalog/sql/Cargo.toml | 1 + crates/catalog/sql/public-api.txt | 2 +- crates/catalog/sql/src/catalog.rs | 447 +++++++++++++++++++----------- 4 files changed, 281 insertions(+), 170 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bab0d84020..082971a0ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3873,6 +3873,7 @@ version = "0.10.1" dependencies = [ "async-trait", "iceberg", + "iceberg-property-macro", "itertools 0.13.0", "regex", "sqlx", diff --git a/crates/catalog/sql/Cargo.toml b/crates/catalog/sql/Cargo.toml index 5861c7807d..e78e4e0ffc 100644 --- a/crates/catalog/sql/Cargo.toml +++ b/crates/catalog/sql/Cargo.toml @@ -32,6 +32,7 @@ repository = { workspace = true } [dependencies] async-trait = { workspace = true } iceberg = { workspace = true } +iceberg-property-macro = { workspace = true } sqlx = { version = "0.8.1", features = ["any"], default-features = false } strum = { workspace = true } tracing = { workspace = true } diff --git a/crates/catalog/sql/public-api.txt b/crates/catalog/sql/public-api.txt index 3a7a93bdae..964a1cae5e 100644 --- a/crates/catalog/sql/public-api.txt +++ b/crates/catalog/sql/public-api.txt @@ -61,7 +61,7 @@ pub fn iceberg_catalog_sql::SqlCatalogBuilder::sql_bind_style(self, sql_bind_sty pub fn iceberg_catalog_sql::SqlCatalogBuilder::uri(self, uri: impl core::convert::Into) -> Self pub fn iceberg_catalog_sql::SqlCatalogBuilder::warehouse_location(self, location: impl core::convert::Into) -> Self impl core::default::Default for iceberg_catalog_sql::SqlCatalogBuilder -pub fn iceberg_catalog_sql::SqlCatalogBuilder::default() -> Self +pub fn iceberg_catalog_sql::SqlCatalogBuilder::default() -> iceberg_catalog_sql::SqlCatalogBuilder impl core::fmt::Debug for iceberg_catalog_sql::SqlCatalogBuilder pub fn iceberg_catalog_sql::SqlCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::catalog::CatalogBuilder for iceberg_catalog_sql::SqlCatalogBuilder diff --git a/crates/catalog/sql/src/catalog.rs b/crates/catalog/sql/src/catalog.rs index 73671f5c4b..e3104dff76 100644 --- a/crates/catalog/sql/src/catalog.rs +++ b/crates/catalog/sql/src/catalog.rs @@ -29,6 +29,7 @@ use iceberg::{ Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, Runtime, TableCommit, TableCreation, TableIdent, }; +use iceberg_property_macro::Properties; use sqlx::any::{AnyPoolOptions, AnyQueryResult, AnyRow, install_default_drivers}; use sqlx::{Any, AnyPool, Column, Executor, Row, Transaction}; @@ -77,62 +78,39 @@ static MAX_CONNECTIONS: u32 = 10; // Default the SQL pool to 10 connections if n static IDLE_TIMEOUT: u64 = 10; // Default the maximum idle timeout per connection to 10s before it is closed static TEST_BEFORE_ACQUIRE: bool = true; // Default the health-check of each connection to enabled prior to returning -fn parse_pool_property( - props: &HashMap, - property: &'static str, - default: T, -) -> Result +/// Parses one pool-property value; the `Properties` derive adds the property-key context. +fn parse_pool_property(value: &str) -> Result where T: FromStr, T::Err: std::error::Error + Send + Sync + 'static, { - props.get(property).map_or(Ok(default), |value| { - value.parse().map_err(|error| { - Error::new( - ErrorKind::DataInvalid, - "Failed to parse SQL catalog pool property", - ) - .with_context("property", property) - .with_context("value", value) - .with_source(error) - }) + value.parse().map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + "Failed to parse SQL catalog pool property", + ) + .with_context("value", value) + .with_source(error) }) } /// Builder for [`SqlCatalog`] -#[derive(Debug)] +#[derive(Debug, Default)] pub struct SqlCatalogBuilder { - config: SqlCatalogConfig, + props: HashMap, storage_factory: Option>, kms_client_factory: Option>, runtime: Option, } -impl Default for SqlCatalogBuilder { - fn default() -> Self { - Self { - config: SqlCatalogConfig { - uri: "".to_string(), - name: "".to_string(), - warehouse_location: "".to_string(), - sql_bind_style: SqlBindStyle::DollarNumeric, - schema_version: None, - props: HashMap::new(), - }, - storage_factory: None, - kms_client_factory: None, - runtime: None, - } - } -} - impl SqlCatalogBuilder { /// Configure the database URI /// /// If `SQL_CATALOG_PROP_URI` has a value set in `props` during `SqlCatalogBuilder::load`, /// that value takes precedence, and the value specified by this method will not be used. pub fn uri(mut self, uri: impl Into) -> Self { - self.config.uri = uri.into(); + self.props + .insert(SQL_CATALOG_PROP_URI.to_string(), uri.into()); self } @@ -141,16 +119,21 @@ impl SqlCatalogBuilder { /// If `SQL_CATALOG_PROP_WAREHOUSE` has a value set in `props` during `SqlCatalogBuilder::load`, /// that value takes precedence, and the value specified by this method will not be used. pub fn warehouse_location(mut self, location: impl Into) -> Self { - self.config.warehouse_location = location.into(); + self.props + .insert(SQL_CATALOG_PROP_WAREHOUSE.to_string(), location.into()); self } /// Configure the bound SQL Statement /// - /// If `SQL_CATALOG_PROP_BIND_STYLE` has a value set in `props` during `SqlCatalogBuilder::load`, - /// that value takes precedence, and the value specified by this method will not be used. + /// If either `SQL_CATALOG_PROP_BIND_STYLE` or the legacy `sql_bind_style` property has a value + /// set in `props` during `SqlCatalogBuilder::load`, that load-time value takes precedence. + /// When both load-time keys are present, `SQL_CATALOG_PROP_BIND_STYLE` takes precedence. pub fn sql_bind_style(mut self, sql_bind_style: SqlBindStyle) -> Self { - self.config.sql_bind_style = sql_bind_style; + self.props.insert( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + sql_bind_style.to_string(), + ); self } @@ -160,7 +143,7 @@ impl SqlCatalogBuilder { /// those values will take precedence. pub fn props(mut self, props: HashMap) -> Self { for (k, v) in props { - self.config.props.insert(k, v); + self.props.insert(k, v); } self } @@ -172,7 +155,7 @@ impl SqlCatalogBuilder { /// If the same key has values set in `props` during `SqlCatalogBuilder::load`, /// those values will take precedence. pub fn prop(mut self, key: impl Into, value: impl Into) -> Self { - self.config.props.insert(key.into(), value.into()); + self.props.insert(key.into(), value.into()); self } } @@ -196,111 +179,135 @@ impl CatalogBuilder for SqlCatalogBuilder { } fn load( - mut self, + self, name: impl Into, props: HashMap, ) -> impl Future> + Send { - for (k, v) in props { - self.config.props.insert(k, v); - } - - if let Some(uri) = self.config.props.remove(SQL_CATALOG_PROP_URI) { - self.config.uri = uri; - } - if let Some(warehouse_location) = self.config.props.remove(SQL_CATALOG_PROP_WAREHOUSE) { - self.config.warehouse_location = warehouse_location; - } - let name = name.into(); - let mut valid_sql_bind_style = true; - - // Accept the preferred `sql.bind-style` key, falling back to the legacy `sql_bind_style`. - let sql_bind_style = self - .config - .props - .remove(SQL_CATALOG_PROP_BIND_STYLE) - .or_else(|| self.config.props.remove(SQL_CATALOG_PROP_BIND_STYLE_LEGACY)); - - // Validate the SQL bind style - if let Some(sql_bind_style) = sql_bind_style { - if let Ok(sql_bind_style) = SqlBindStyle::from_str(&sql_bind_style) { - self.config.sql_bind_style = sql_bind_style; - } else { - valid_sql_bind_style = false; + async move { + if name.trim().is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + "Catalog name cannot be empty", + )); } - } - // Parse the requested schema version up front so invalid values fail fast rather than - // silently falling back to V0. - let mut valid_schema_version = true; - if let Some(schema_version) = self.config.props.remove(SQL_CATALOG_PROP_SCHEMA_VERSION) { - match SchemaVersion::from_str(&schema_version) { - Ok(schema_version) => self.config.schema_version = Some(schema_version), - Err(_) => valid_schema_version = false, + let load_overrides_bind_style = props.contains_key(SQL_CATALOG_PROP_BIND_STYLE) + || props.contains_key(SQL_CATALOG_PROP_BIND_STYLE_LEGACY); + let mut merged_props = self.props; + if load_overrides_bind_style { + // Treat the preferred and legacy names as aliases so either load-time spelling + // overrides a value configured on the builder. + merged_props.remove(SQL_CATALOG_PROP_BIND_STYLE); + merged_props.remove(SQL_CATALOG_PROP_BIND_STYLE_LEGACY); } - } + merged_props.extend(props); + let catalog_properties = SqlCatalogProperties::from_properties(&merged_props)?; - let valid_name = !name.trim().is_empty(); + // The catalog URI is consumed above and may contain database credentials. Do not + // expose this security-sensitive value to FileIO or KMS extension points. + merged_props.remove(SQL_CATALOG_PROP_URI); - async move { - if !valid_name { - Err(Error::new( - ErrorKind::DataInvalid, - "Catalog name cannot be empty", - )) - } else if !valid_sql_bind_style { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "`{}` values are valid only if they're `{}` or `{}`", - SQL_CATALOG_PROP_BIND_STYLE, - SqlBindStyle::DollarNumeric, - SqlBindStyle::QMark - ), - )) - } else if !valid_schema_version { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "`{}` values are valid only if they're `{}` or `{}`", - SQL_CATALOG_PROP_SCHEMA_VERSION, - SchemaVersion::V0, - SchemaVersion::V1 - ), - )) - } else { - self.config.name = name; - let runtime = match self.runtime { - Some(rt) => rt, - None => Runtime::try_current()?, - }; - let kms_client = match self.kms_client_factory { - Some(factory) => Some(factory.create_kms_client(&self.config.props).await?), - None => None, - }; - SqlCatalog::new(self.config, self.storage_factory, runtime, kms_client).await - } + let runtime = match self.runtime { + Some(rt) => rt, + None => Runtime::try_current()?, + }; + let kms_client = match self.kms_client_factory { + Some(factory) => Some(factory.create_kms_client(&merged_props).await?), + None => None, + }; + SqlCatalog::new( + name, + catalog_properties, + merged_props, + self.storage_factory, + runtime, + kms_client, + ) + .await } } } -/// A struct representing the SQL catalog configuration. -/// -/// This struct contains various parameters that are used to configure a SQL catalog, -/// such as the database URI, warehouse location, and file I/O settings. -/// You are required to provide a `SqlBindStyle`, which determines how SQL statements will be bound to values in the catalog. -/// The options available for this parameter include: -/// - `SqlBindStyle::DollarNumeric`: Binds SQL statements using `$1`, `$2`, etc., as placeholders. This is for PostgreSQL databases. -/// - `SqlBindStyle::QuestionMark`: Binds SQL statements using `?` as a placeholder. This is for MySQL and SQLite databases. -#[derive(Debug)] -struct SqlCatalogConfig { +fn parse_sql_bind_style( + properties: &HashMap, + key: &str, + additional_keys: &[&str], + default: SqlBindStyle, +) -> Result { + let configured_value = properties.get(key).map(|value| (key, value)).or_else(|| { + additional_keys + .iter() + .find_map(|alt_key| properties.get(*alt_key).map(|value| (*alt_key, value))) + }); + + configured_value.map_or(Ok(default), |(configured_key, value)| { + SqlBindStyle::from_str(value).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "`{}` values are valid only if they're `{}` or `{}`", + configured_key, + SqlBindStyle::DollarNumeric, + SqlBindStyle::QMark + ), + ) + }) + }) +} + +fn parse_schema_version(value: &str) -> Result { + SchemaVersion::from_str(value).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "`{}` values are valid only if they're `{}` or `{}`", + SQL_CATALOG_PROP_SCHEMA_VERSION, + SchemaVersion::V0, + SchemaVersion::V1 + ), + ) + }) +} + +#[derive(Debug, Properties)] +pub(crate) struct SqlCatalogProperties { + #[property(key = SQL_CATALOG_PROP_URI, default = "")] uri: String, - name: String, + #[property(key = SQL_CATALOG_PROP_WAREHOUSE, default = "")] warehouse_location: String, + #[property( + key = SQL_CATALOG_PROP_BIND_STYLE, + additional_keys = [SQL_CATALOG_PROP_BIND_STYLE_LEGACY], + default = SqlBindStyle::DollarNumeric, + parse_properties_with = parse_sql_bind_style + )] sql_bind_style: SqlBindStyle, + #[property( + key = SQL_CATALOG_PROP_SCHEMA_VERSION, + default = None, + parse_with = parse_schema_version + )] schema_version: Option, - props: HashMap, + #[property( + key = "pool.max-connections", + default = MAX_CONNECTIONS, + parse_with = parse_pool_property + )] + max_connections: u32, + #[property( + key = "pool.idle-timeout", + default = IDLE_TIMEOUT, + parse_with = parse_pool_property + )] + idle_timeout: u64, + #[property( + key = "pool.test-before-acquire", + default = TEST_BEFORE_ACQUIRE, + parse_with = parse_pool_property + )] + test_before_acquire: bool, } #[derive(Debug)] @@ -310,10 +317,9 @@ struct SqlCatalogConfig { /// Catalogs can opt-in to automatic migration by configuring the `sql.schema-version` catalog property. pub struct SqlCatalog { name: String, + properties: SqlCatalogProperties, connection: AnyPool, - warehouse_location: String, fileio: FileIO, - sql_bind_style: SqlBindStyle, runtime: Runtime, kms_client: Option>, schema_version: SchemaVersion, @@ -390,7 +396,9 @@ pub enum SqlBindStyle { impl SqlCatalog { /// Create new sql catalog instance async fn new( - config: SqlCatalogConfig, + name: String, + properties: SqlCatalogProperties, + props: HashMap, storage_factory: Option>, runtime: Runtime, kms_client: Option>, @@ -401,27 +409,16 @@ impl SqlCatalog { "StorageFactory must be provided for SqlCatalog. Use `with_storage_factory` to configure it.", ) })?; - // Forward catalog props so storage-backend keys reach the FileIO. - // Unrecognized keys are ignored by backends. - let fileio = FileIOBuilder::new(factory) - .with_props(config.props.clone()) - .build(); - install_default_drivers(); - let max_connections = - parse_pool_property(&config.props, "pool.max-connections", MAX_CONNECTIONS)?; - let idle_timeout = parse_pool_property(&config.props, "pool.idle-timeout", IDLE_TIMEOUT)?; - let test_before_acquire = parse_pool_property( - &config.props, - "pool.test-before-acquire", - TEST_BEFORE_ACQUIRE, - )?; + // Forward the complete property map so storage-backend keys reach FileIO. + // Unrecognized keys are ignored by backends. + let fileio = FileIOBuilder::new(factory).with_props(props).build(); let pool = AnyPoolOptions::new() - .max_connections(max_connections) - .idle_timeout(Duration::from_secs(idle_timeout)) - .test_before_acquire(test_before_acquire) - .connect(&config.uri) + .max_connections(properties.max_connections) + .idle_timeout(Duration::from_secs(properties.idle_timeout)) + .test_before_acquire(properties.test_before_acquire) + .connect(&properties.uri) .await .map_err(from_sqlx_error)?; @@ -452,7 +449,7 @@ impl SqlCatalog { .map_err(from_sqlx_error)?; let detected_schema_version = SchemaVersion::detect(&pool).await?; - let expected_schema_version = config.schema_version; + let expected_schema_version = properties.schema_version; // Detect schema by describing columns. If expected is configured then automigrate, otherwise gracefully support older schemas. let schema_version = match (detected_schema_version, expected_schema_version) { @@ -498,11 +495,10 @@ impl SqlCatalog { }; Ok(SqlCatalog { - name: config.name.to_owned(), + name, + properties, connection: pool, - warehouse_location: config.warehouse_location, fileio, - sql_bind_style: config.sql_bind_style, runtime, kms_client, schema_version, @@ -511,7 +507,7 @@ impl SqlCatalog { /// SQLX Any does not implement PostgresSQL bindings, so we have to do this. fn replace_placeholders(&self, query: &str) -> String { - match self.sql_bind_style { + match self.properties.sql_bind_style { SqlBindStyle::DollarNumeric => { let mut count = 1; query @@ -1047,7 +1043,7 @@ impl Catalog for SqlCatalog { None => { format!( "{}/{}", - self.warehouse_location.clone(), + &self.properties.warehouse_location, namespace.join("/") ) } @@ -1244,9 +1240,10 @@ mod tests { use tempfile::TempDir; use crate::catalog::{ - CATALOG_FIELD_RECORD_TYPE, CATALOG_TABLE_NAME, NAMESPACE_LOCATION_PROPERTY_KEY, - NAMESPACE_TABLE_NAME, SQL_CATALOG_PROP_BIND_STYLE, SQL_CATALOG_PROP_BIND_STYLE_LEGACY, - SQL_CATALOG_PROP_SCHEMA_VERSION, SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, + CATALOG_FIELD_RECORD_TYPE, CATALOG_TABLE_NAME, IDLE_TIMEOUT, MAX_CONNECTIONS, + NAMESPACE_LOCATION_PROPERTY_KEY, NAMESPACE_TABLE_NAME, SQL_CATALOG_PROP_BIND_STYLE, + SQL_CATALOG_PROP_BIND_STYLE_LEGACY, SQL_CATALOG_PROP_SCHEMA_VERSION, SQL_CATALOG_PROP_URI, + SQL_CATALOG_PROP_WAREHOUSE, SqlCatalogProperties, TEST_BEFORE_ACQUIRE, }; use crate::{SchemaVersion, SqlBindStyle, SqlCatalog, SqlCatalogBuilder}; @@ -1257,6 +1254,85 @@ mod tests { temp_dir.path().to_str().unwrap().to_string() } + #[test] + fn test_catalog_properties() { + let properties = SqlCatalogProperties::from_properties(&HashMap::from([ + ( + SQL_CATALOG_PROP_URI.to_string(), + "sqlite://catalog".to_string(), + ), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + "/warehouse".to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ( + SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(), + "V1".to_string(), + ), + ("pool.max-connections".to_string(), "5".to_string()), + ("pool.idle-timeout".to_string(), "20".to_string()), + ("pool.test-before-acquire".to_string(), "false".to_string()), + ])) + .unwrap(); + + assert_eq!(properties.uri, "sqlite://catalog"); + assert_eq!(properties.warehouse_location, "/warehouse"); + assert_eq!(properties.sql_bind_style, SqlBindStyle::QMark); + assert_eq!(properties.schema_version, Some(SchemaVersion::V1)); + assert_eq!(properties.max_connections, 5); + assert_eq!(properties.idle_timeout, 20); + assert!(!properties.test_before_acquire); + } + + #[test] + fn test_catalog_properties_defaults() { + let properties = SqlCatalogProperties::from_properties(&HashMap::new()).unwrap(); + + assert_eq!(properties.uri, ""); + assert_eq!(properties.warehouse_location, ""); + assert_eq!(properties.sql_bind_style, SqlBindStyle::DollarNumeric); + assert_eq!(properties.schema_version, None); + assert_eq!(properties.max_connections, MAX_CONNECTIONS); + assert_eq!(properties.idle_timeout, IDLE_TIMEOUT); + assert_eq!(properties.test_before_acquire, TEST_BEFORE_ACQUIRE); + } + + #[test] + fn test_preferred_bind_style_key_takes_precedence_over_legacy_key() { + let properties = SqlCatalogProperties::from_properties(&HashMap::from([ + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + SqlBindStyle::DollarNumeric.to_string(), + ), + ])) + .unwrap(); + + assert_eq!(properties.sql_bind_style, SqlBindStyle::QMark); + } + + #[test] + fn test_invalid_legacy_bind_style_error_names_configured_key() { + let error = SqlCatalogProperties::from_properties(&HashMap::from([( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + "invalid".to_string(), + )])) + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!( + error.message(), + "`sql_bind_style` values are valid only if they're `DollarNumeric` or `QMark`" + ); + } + fn to_set(vec: Vec) -> HashSet { HashSet::from_iter(vec) } @@ -1482,7 +1558,10 @@ mod tests { "iceberg", HashMap::from_iter([ (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri), - (SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse_location.clone(), + ), ("s3.region".to_string(), "us-east-1".to_string()), ("hf.token".to_string(), "hf_test_token".to_string()), ]), @@ -1491,6 +1570,11 @@ mod tests { .unwrap(); let props = catalog.fileio.config().props(); + assert_eq!(props.get(SQL_CATALOG_PROP_URI), None); + assert_eq!( + props.get(SQL_CATALOG_PROP_WAREHOUSE), + Some(&warehouse_location) + ); assert_eq!(props.get("s3.region"), Some(&"us-east-1".to_string())); assert_eq!(props.get("hf.token"), Some(&"hf_test_token".to_string())); } @@ -1511,8 +1595,8 @@ mod tests { assert!(catalog.is_ok()); let catalog = catalog.unwrap(); - assert!(catalog.warehouse_location == warehouse_location); - assert!(catalog.sql_bind_style == SqlBindStyle::QMark); + assert!(catalog.properties.warehouse_location == warehouse_location); + assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark); } /// Overwriting an sqlite database with a non-existent path causes @@ -1589,8 +1673,33 @@ mod tests { assert!(catalog.is_ok()); let catalog = catalog.unwrap(); - assert!(catalog.warehouse_location == warehouse_location); - assert!(catalog.sql_bind_style == SqlBindStyle::QMark); + assert!(catalog.properties.warehouse_location == warehouse_location); + assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark); + } + + #[tokio::test] + async fn test_load_legacy_bind_style_overrides_builder_method() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .sql_bind_style(SqlBindStyle::DollarNumeric) + .load( + "iceberg", + HashMap::from([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri), + (SQL_CATALOG_PROP_WAREHOUSE.to_string(), temp_path()), + ( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]), + ) + .await + .unwrap(); + + assert_eq!(catalog.properties.sql_bind_style, SqlBindStyle::QMark); } /// values assigned via props take precedence @@ -1634,8 +1743,8 @@ mod tests { assert!(catalog.is_ok()); let catalog = catalog.unwrap(); - assert!(catalog.warehouse_location == warehouse_location); - assert!(catalog.sql_bind_style == SqlBindStyle::QMark); + assert!(catalog.properties.warehouse_location == warehouse_location); + assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark); } /// values assigned via props take precedence @@ -1673,8 +1782,8 @@ mod tests { assert!(catalog.is_ok()); let catalog = catalog.unwrap(); - assert!(catalog.warehouse_location == warehouse_location); - assert!(catalog.sql_bind_style == SqlBindStyle::QMark); + assert!(catalog.properties.warehouse_location == warehouse_location); + assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark); } /// invalid value for `SqlBindStyle` causes catalog creation to fail @@ -2883,6 +2992,6 @@ mod tests { .await .expect("legacy sql_bind_style key should still be accepted"); - assert_eq!(catalog.sql_bind_style, SqlBindStyle::QMark); + assert_eq!(catalog.properties.sql_bind_style, SqlBindStyle::QMark); } } From d2b2f7ae43e677c13b45500bd7ea953cb9a961ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:18:17 +0000 Subject: [PATCH 60/74] chore(deps): Bump taiki-e/install-action from 2.87.2 to 2.87.8 (#3217) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.87.2 to 2.87.8. - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/1ed6d7be6168f6c9046541087ff549b6bc581fdf...d438492cf8a250514fa2d34b30bc3c0dc37c65ff) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.87.8 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34cf9a7ed8..f9fce8c20c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: - name: Install lint tools # Keep versions in sync with the local install targets in Makefile. - uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2 + uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: taplo@0.9.3,cargo-machete@0.7.0 @@ -235,7 +235,7 @@ jobs: - name: Install cargo-nextest if: matrix.test-suite.name == 'default' - uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2 + uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 with: tool: cargo-nextest From 99c57c953f1a9fce4785f5d77cedb65f2f8a362b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:18:53 +0000 Subject: [PATCH 61/74] chore(deps): Bump the arrow-parquet group across 1 directory with 10 updates (#3209) Bumps the arrow-parquet group with 2 updates in the / directory: [arrow](https://github.com/apache/arrow-rs) and [parquet](https://github.com/apache/arrow-rs). Updates `arrow` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `arrow-arith` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `arrow-array` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `arrow-buffer` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `arrow-cast` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `arrow-ord` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `arrow-schema` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `arrow-select` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `arrow-string` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) Updates `parquet` from 59.2.0 to 59.3.0 - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/59.3.0/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/59.2.0...59.3.0) --- updated-dependencies: - dependency-name: arrow dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: arrow-arith dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: arrow-array dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: arrow-buffer dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: arrow-cast dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: arrow-ord dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: arrow-schema dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: arrow-select dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: arrow-string dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet - dependency-name: parquet dependency-version: 59.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: arrow-parquet ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 68 +++++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 082971a0ee..069acd0d9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,9 +227,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" dependencies = [ "arrow-arith", "arrow-array", @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" dependencies = [ "arrow-array", "arrow-buffer", @@ -263,9 +263,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" dependencies = [ "ahash", "arrow-buffer", @@ -307,9 +307,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" dependencies = [ "bytes", "half", @@ -319,9 +319,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" +checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86" dependencies = [ "arrow-array", "arrow-cast", @@ -356,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" dependencies = [ "arrow-buffer", "arrow-schema", @@ -369,9 +369,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" dependencies = [ "arrow-array", "arrow-buffer", @@ -385,9 +385,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" +checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1" dependencies = [ "arrow-array", "arrow-buffer", @@ -410,9 +410,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" dependencies = [ "arrow-array", "arrow-buffer", @@ -423,9 +423,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c196ecc25b3a8dcbc1d842f2619cee653dcfa2fb8b56a291bc0481c3cf5c3821" +checksum = "801aed7e607dbcc60ed6359025383b27a490327d4282ba87a9a3b78e25959c9a" dependencies = [ "arrow-array", "arrow-data", @@ -435,9 +435,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" dependencies = [ "arrow-array", "arrow-buffer", @@ -448,9 +448,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" +checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -459,9 +459,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" dependencies = [ "ahash", "arrow-array", @@ -473,9 +473,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" dependencies = [ "arrow-array", "arrow-buffer", @@ -1484,7 +1484,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -5425,9 +5425,9 @@ dependencies = [ [[package]] name = "parquet" -version = "59.2.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" +checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73" dependencies = [ "ahash", "arrow-array", @@ -8542,7 +8542,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] From 147d09fbc0a1ca0d4e24bc5a37ce711d1718addf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:31:20 +0000 Subject: [PATCH 62/74] chore(deps-dev): Bump pyiceberg from 0.11.1 to 0.12.0 in /bindings/python (#3208) chore(deps-dev): Bump pyiceberg in /bindings/python Bumps [pyiceberg](https://github.com/apache/iceberg-python) from 0.11.1 to 0.12.0. - [Release notes](https://github.com/apache/iceberg-python/releases) - [Commits](https://github.com/apache/iceberg-python/compare/pyiceberg-0.11.1...pyiceberg-0.12.0) --- updated-dependencies: - dependency-name: pyiceberg dependency-version: 0.12.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- bindings/python/uv.lock | 67 +++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock index 0f45de4b99..31569dc790 100644 --- a/bindings/python/uv.lock +++ b/bindings/python/uv.lock @@ -619,7 +619,7 @@ wheels = [ [[package]] name = "pyiceberg" -version = "0.11.1" +version = "0.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -635,36 +635,43 @@ dependencies = [ { name = "tenacity" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ce/f0/7616676603fdbd05ab97816337a9b31be08a5f9e1ffd636260812b217e0f/pyiceberg-0.11.1.tar.gz", hash = "sha256:366fe0d5a74e3cf1d4e7cbf3c49e308da60e7835ea268667be9185388f05d7a5", size = 1076075, upload-time = "2026-03-03T00:10:27.61Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/08/bde71e0bbcf1a62c92d7fa457b508691596c65fa7e52c1982c78c461cd1c/pyiceberg-0.12.0.tar.gz", hash = "sha256:19f165d298054f9436108691098b60fa0fa99d0eff5fb884700c43b29334a39d", size = 1212830, upload-time = "2026-09-01T17:28:42.378Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/40/ea9cc312ea8ad76d94c57f921be19256c3d07354056c68e64601ff43f98d/pyiceberg-0.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:89c347170a691e3b8d072519f36790b9c19a2263ec0af0144c873994ecfe55eb", size = 532405, upload-time = "2026-03-03T00:09:49.364Z" }, - { url = "https://files.pythonhosted.org/packages/97/e9/fbcd52d19aa2a362ba7fd971b1721c57f8c0fd37b5bb418ac283dde9637e/pyiceberg-0.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:138983e96a5c473aae1e1cd6143aacf02ccc9bbbf6180f15fa2588f99531188b", size = 533282, upload-time = "2026-03-03T00:09:50.712Z" }, - { url = "https://files.pythonhosted.org/packages/1d/43/a0722863f821e96eed8ff5753a7146dd5b1537668d6684ac650a48647669/pyiceberg-0.11.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e08c268e12d54ea629a1f123bf7cb9587bdd57fd7bcdac22394dc6272a18668f", size = 708618, upload-time = "2026-03-03T00:09:52.427Z" }, - { url = "https://files.pythonhosted.org/packages/72/76/3f4aa7f8fe99292b8074720f5726040f6c7528209666444f1d6166c17aeb/pyiceberg-0.11.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23ed91f8d2fa75a109708338f80a7addd2f37c93a8b0e6c9d220f1427939b81f", size = 705581, upload-time = "2026-03-03T00:09:54.389Z" }, - { url = "https://files.pythonhosted.org/packages/27/12/85543a838b2b3f4a74dfd8de470f6d88f6ba5d287231b6ea4e8765df04f6/pyiceberg-0.11.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b4b91f1b1bac135c077326d30a1876f917e6c36b844a2e329b148fb9cb7aa660", size = 704197, upload-time = "2026-03-03T00:09:55.809Z" }, - { url = "https://files.pythonhosted.org/packages/f6/20/7a37d331722a4cfda0e782ec583aa28af412f8564ac937733eac440aa7b0/pyiceberg-0.11.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7770e7bdee34549e8baf28b8626e6552de7894d869465a7c95fd9af33b4cbeb0", size = 703982, upload-time = "2026-03-03T00:09:57.098Z" }, - { url = "https://files.pythonhosted.org/packages/65/16/61c2af5ab30f61f3b523346d109bc52b035bb26ecdfa58524c63fac190c4/pyiceberg-0.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:ccac408873f65b8954858ef08d4b098841301e232afc1b14e3eada489d303a7f", size = 530720, upload-time = "2026-03-03T00:09:58.548Z" }, - { url = "https://files.pythonhosted.org/packages/62/f7/3b7fee2ecc021f0526f23ef4ae5dcc8e0ed26062c35890ad25d39c53fb3b/pyiceberg-0.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba98d6a41ec0b7c81dd85d764f15653d6abbbbd69d92630677c43f92dd50d924", size = 532406, upload-time = "2026-03-03T00:09:59.647Z" }, - { url = "https://files.pythonhosted.org/packages/94/25/324030b13d91b7b564fb7342bf3fcdbf76eed2672964b273b156bf84d6e5/pyiceberg-0.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6400774e6820760eb6c322f6feace43fe7267deb9f8d508f10bf258887a9c4d5", size = 533368, upload-time = "2026-03-03T00:10:01.264Z" }, - { url = "https://files.pythonhosted.org/packages/1c/83/6a43d06a079292c4fc7815b4de3e90a05ded90031c35d0a1b037659f722b/pyiceberg-0.11.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1663d79fc8400903992c63f79b3908b9298c623138e8929bf36c559231e082d3", size = 722886, upload-time = "2026-03-03T00:10:02.558Z" }, - { url = "https://files.pythonhosted.org/packages/ce/5c/53807036b63bb810f2c56b4b5576e79b721ba93f1c16bd0dd49ecfe41055/pyiceberg-0.11.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:856c7fca5ed780ed44f60bceb92d6b311ebc008a2249415b8f6045201d4f5530", size = 721212, upload-time = "2026-03-03T00:10:03.694Z" }, - { url = "https://files.pythonhosted.org/packages/db/11/65e25d6016e3844c516c9f04041853115711f64af1ee184a2320c9ceab4c/pyiceberg-0.11.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6cf94756fb6a822d20a5a64f44840e6633ebf8b1deb3ce01057bff1cc03b01c2", size = 717978, upload-time = "2026-03-03T00:10:05.011Z" }, - { url = "https://files.pythonhosted.org/packages/52/0b/33b4ea9dc7f0c496900f5fc6da79e8587e94b88e2244ff02b786016cc649/pyiceberg-0.11.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8bd52c1891ae74cee21a4ebe8325953310a2e0af5352d70f47f5461422fcce2d", size = 718747, upload-time = "2026-03-03T00:10:06.269Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b6/5e133c435efc577afea5be303d7123264a5176a3ce1e2d3dc3a691049eaf/pyiceberg-0.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:65a7ad892a570045b0de2db6af17119162880aebc05a0c125ce2db7dab36f17e", size = 531081, upload-time = "2026-03-03T00:10:07.352Z" }, - { url = "https://files.pythonhosted.org/packages/8f/84/a140466b7e0841207e6b77042e03d4ab3a4f9d47e00f0bbbcc5420792bbb/pyiceberg-0.11.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cd423b8ee2f75fc9db09158875abe5e2c952a26ae5e521c3265ab2f9d3511ddf", size = 532981, upload-time = "2026-03-03T00:10:08.906Z" }, - { url = "https://files.pythonhosted.org/packages/17/10/6bedd784010f707680ffd0606d4d11394cf915f4f9f54ae16e8007e00ad4/pyiceberg-0.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e273242cdca56029af694d7ce18075d47a74d034326d663ff6dd2655a6f44825", size = 533188, upload-time = "2026-03-03T00:10:10.086Z" }, - { url = "https://files.pythonhosted.org/packages/f1/a3/79db617c3cffc963efa8a332707079d3f22fd58067b31a208d358dd89b39/pyiceberg-0.11.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b347d3cc8510f8fbe191956fcda7da372ebb3302789acefca08e352345959003", size = 729546, upload-time = "2026-03-03T00:10:11.413Z" }, - { url = "https://files.pythonhosted.org/packages/06/64/acc11d230c33817bced80d9d947bb49e7bb3a429d76d906523e3df86faf8/pyiceberg-0.11.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bba3a35b4648694783aeae5b77c235a57191c8b1b375c8602b03ae56a6cf4fe7", size = 730263, upload-time = "2026-03-03T00:10:13.283Z" }, - { url = "https://files.pythonhosted.org/packages/8d/1a/fb067d5150c7309fbf5dd126c648a6afed6259e7bc924ba3c65d0f87a333/pyiceberg-0.11.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0f958cbca18d05846e3081dfff8575e73d45595441d659847479656dc76f91d", size = 724064, upload-time = "2026-03-03T00:10:14.55Z" }, - { url = "https://files.pythonhosted.org/packages/c1/71/103fdba5b144d55f3bb07347893737cc1d8fd71308108a77b7817c92c544/pyiceberg-0.11.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8c62636a1e9d8a1fc74ffb70383939b9cd93f2c9ee8e12015a50dd75c98a989e", size = 727239, upload-time = "2026-03-03T00:10:16.204Z" }, - { url = "https://files.pythonhosted.org/packages/18/c3/4db64429304c58c039f8e842cd37a9a1c472f596c2868ed2a5d2907b17ed/pyiceberg-0.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:1d6b6f0c1e7dd8357f1ba56524bfc870d04ad3c00979db291784a7145497ad3b", size = 531309, upload-time = "2026-03-03T00:10:17.561Z" }, - { url = "https://files.pythonhosted.org/packages/35/4c/a122d80d98cb6125d87024681263406433f0c25c699d503f5633521e6809/pyiceberg-0.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b7ec5db19feab98a31fcd5caccf4a9a4e83f96933d1ca393ba7aea665710c2bb", size = 532644, upload-time = "2026-03-03T00:10:18.574Z" }, - { url = "https://files.pythonhosted.org/packages/10/94/9a8fa5fc580e6dccd34bbbf51e7658cd7b49540e2458783addeff5e22a91/pyiceberg-0.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cec0616d2ba6e7dda6327089a2f34ec723aa9ac2c389857ef0b83f65fb135dd6", size = 532787, upload-time = "2026-03-03T00:10:19.656Z" }, - { url = "https://files.pythonhosted.org/packages/b3/ab/ab7c88828bc17d77dbbc5a765419dfec2135629e1d74cdd0762cd38ad867/pyiceberg-0.11.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ddb360da76c62c7c23ec3da40e1af48e6712a563905fea2d1a8911ff7a3b6c4d", size = 722202, upload-time = "2026-03-03T00:10:21.012Z" }, - { url = "https://files.pythonhosted.org/packages/df/38/079cf1c0bf86da315472a926eec0dba10135f43374a2e267336eb98d8c76/pyiceberg-0.11.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d8790f420ebc484236017edba59182cf2a21bd3e4224a0bd0760a9c7268e96a", size = 724037, upload-time = "2026-03-03T00:10:22.176Z" }, - { url = "https://files.pythonhosted.org/packages/08/6b/08eaef477debb110438d943ef3f5985096f660ccb735d6344701cbd075a9/pyiceberg-0.11.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ae27ba4d37925d5b2cff192acaa70c8bb114d632bbc527cc91fea0370702b866", size = 716035, upload-time = "2026-03-03T00:10:23.789Z" }, - { url = "https://files.pythonhosted.org/packages/0b/59/7671d6a630ab1d85c6e7ca8ddf438dc63a0b0dd183bc4be69bf25c0fa5f6/pyiceberg-0.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db66a4e0fdfbf4090631d59c3f65e960d9a5561e9259f6f3993cbe91e396837e", size = 720887, upload-time = "2026-03-03T00:10:24.824Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2b/5c8ad37807efaedb14b20f01f36462684468c80da5b74f4018fb4c1804b5/pyiceberg-0.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:eb3a0a3e630ee89758eb96b39b456f4697732351fb0c080e9498ea578f9b71f9", size = 530923, upload-time = "2026-03-03T00:10:26.196Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a4/32d35527d48b3d569d7cc0a4319a52c431d91e0a82f8855059167426b741/pyiceberg-0.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f7462f4777bcda2af0a7c270a51525542a79ff37cf1334a82e0dd80717aa92", size = 567107, upload-time = "2026-09-01T17:27:57.111Z" }, + { url = "https://files.pythonhosted.org/packages/f7/9f/5d1c08d172a5ed6f2eef466435576fb11efdcdb04c8d7eb0de1eb5d9363d/pyiceberg-0.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:97132565255b94831ce858837cc333a1ac837a116e1ce233299e0b6a67f82418", size = 568086, upload-time = "2026-09-01T17:27:58.794Z" }, + { url = "https://files.pythonhosted.org/packages/02/01/79e2fe2b35f2cff510a795fffc2eb9f82c84a96a3e29ed3fca40bc3a1f63/pyiceberg-0.12.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82e7164eb23c46b567d7fafb38058d7ca13aec5d4ef59d0ee701cfdeda004dfe", size = 761523, upload-time = "2026-09-01T17:28:00.02Z" }, + { url = "https://files.pythonhosted.org/packages/e7/34/6e635511ef60d26f036efbea60bb0820ede0ca311743a4f6bac085a0f394/pyiceberg-0.12.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c6484af01afa37a3f3cfb299b4fd8b14b138f95955b57cb01c10acd88171d28", size = 758400, upload-time = "2026-09-01T17:28:01.291Z" }, + { url = "https://files.pythonhosted.org/packages/d2/69/d3c12cb3ff64e1616c538651731bed18a7734e7f51bcf8f5f5aa4296072b/pyiceberg-0.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b54c151610a4f9e198fa8c13c881c4c4d4f1beaa4571023dd79428aa1bbe7f5e", size = 757281, upload-time = "2026-09-01T17:28:02.366Z" }, + { url = "https://files.pythonhosted.org/packages/ba/f7/fe179589ba419a082e7c29c995cf7bbe6d69080d1feb6c0806bbf1551d44/pyiceberg-0.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5a667fd37415626fbfc7b01e373ef66a09f0ddd185875a11e868a5ef128da1f6", size = 757202, upload-time = "2026-09-01T17:28:03.688Z" }, + { url = "https://files.pythonhosted.org/packages/7c/5f/735d67ab9ced4dfc1396755a19c4ec286fc78702f81ad5c8720ec407d19e/pyiceberg-0.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:2c0bcdff87f92b35c4374a6f680b3c6e25e5a87e40fa61df49a563dc8ae13e44", size = 564549, upload-time = "2026-09-01T17:28:05.169Z" }, + { url = "https://files.pythonhosted.org/packages/5e/07/aab0770651e9dd70902ff170d7db3875caa2f53c0df1103035022721d54d/pyiceberg-0.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ae611854967d691f3158ae06407622072a9c442e29f5fba073ecbedb4d0b8b66", size = 566935, upload-time = "2026-09-01T17:28:06.564Z" }, + { url = "https://files.pythonhosted.org/packages/6a/11/c14909196946ab59677d8c814bd9d811691b62a794bb27895d406dd34119/pyiceberg-0.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:63929b5664512c67bb89603c5ca8c568a08b4a9922ec86675b339214d362ea81", size = 567919, upload-time = "2026-09-01T17:28:07.642Z" }, + { url = "https://files.pythonhosted.org/packages/88/05/33f6ef4315e6210dcd7e98dbd4c367390ca24b71820f782df0fb5df49aee/pyiceberg-0.12.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cd913759564d08444567690c27a187b408d06e6c82a17ba4e4f98aaf90031e86", size = 773938, upload-time = "2026-09-01T17:28:08.672Z" }, + { url = "https://files.pythonhosted.org/packages/f0/38/11579d59d3d91288e4b805d13642417ebadae3548c554b2474cb8229b616/pyiceberg-0.12.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2dec70218c8e81f1630b9d7184b760cd161eff936127745355ba94100a4e7380", size = 771959, upload-time = "2026-09-01T17:28:10.099Z" }, + { url = "https://files.pythonhosted.org/packages/84/77/17168e83f5216a57b74c1c64875f5469239d16ea1ad60902aed7a2fdc2d3/pyiceberg-0.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:29fb5032276aaac1ad01f98125622012067be075cc70053a0dbd0d61e4a6e8e5", size = 770200, upload-time = "2026-09-01T17:28:11.432Z" }, + { url = "https://files.pythonhosted.org/packages/25/0c/88fcbf58f0c659b7f714ceed56ffd42b26f85fafbf5eea61056d9c10e654/pyiceberg-0.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0b432f97a7774a211980f8d135c12a75221d0696204a0dc67f9b750a5c1a3332", size = 770216, upload-time = "2026-09-01T17:28:12.637Z" }, + { url = "https://files.pythonhosted.org/packages/60/80/5e3c42eb35c88ed5b73bae31e1fc0cca69f7c09a604aafc09bec1feab68d/pyiceberg-0.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb4d04f93c1b98c19c365dc2bc8115e8396a1b2e1eb99ec9e04ade262c82c565", size = 564293, upload-time = "2026-09-01T17:28:13.888Z" }, + { url = "https://files.pythonhosted.org/packages/b1/45/9fc0692dd081ab34e53731b0cf3eba0f948e5a54948840fa06a4ed4cbf9b/pyiceberg-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0079c44d065fc70df09deb03dc7297314c31b8d9708559c0bec2b8f851ffbf21", size = 567622, upload-time = "2026-09-01T17:28:15.199Z" }, + { url = "https://files.pythonhosted.org/packages/0e/67/b11334fe6af2a5729bfd30c0e03dee78887e9874a634888aa81d91dc82e9/pyiceberg-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:baf35f45ed5ee1a14c8db8264de31fb942b9d0d21913262f0d444d2df45e8176", size = 568214, upload-time = "2026-09-01T17:28:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/1d/24/eccd190e358514e7e0d9a5c7591e44e71eb0e2ab5cff6cb90adb9ecaa963/pyiceberg-0.12.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b20c36d08b4b12da572b10c20594be78eee7e51845040443c45231d2ad57c28a", size = 779432, upload-time = "2026-09-01T17:28:17.775Z" }, + { url = "https://files.pythonhosted.org/packages/ed/75/046692b5ae4330d251974a428fdd82c9d7840715580d332c7cbcd65a13db/pyiceberg-0.12.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17dca4377c76b7b047a2e807f009595b8f0b090c792a84ecb427c2fa683a96e2", size = 781395, upload-time = "2026-09-01T17:28:18.906Z" }, + { url = "https://files.pythonhosted.org/packages/5f/38/f8b0dc8cbc53459c6780a3ebf78382960c78fd80db54d0eeb3f7a63c9c7f/pyiceberg-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5902178a7d46bc4b783a4026c474178a8cb4c9c413b47cdb2e520df4082ed255", size = 773096, upload-time = "2026-09-01T17:28:20.236Z" }, + { url = "https://files.pythonhosted.org/packages/d9/4e/ef4265f3b7108d1591ff233edf3ef6ca50d2639026b4a1cb62ddda3398bd/pyiceberg-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cbb9f060d170b1b072e2ba2e821f5c76302e5731452f37fa63e104334f3feba1", size = 778402, upload-time = "2026-09-01T17:28:21.79Z" }, + { url = "https://files.pythonhosted.org/packages/f9/2c/858c329a4e93897568ab73e95892dde0bc32bf4a76a6133096ef6b660854/pyiceberg-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:0fcce46f5633491b50ebf8ed94fbf5c8d3b6bed76902cf28361368e9169e16f2", size = 564568, upload-time = "2026-09-01T17:28:23.048Z" }, + { url = "https://files.pythonhosted.org/packages/44/7d/c04a65b08ba272bfcbb31638222d71a9f1c7f12c0d8b659530e14817afaf/pyiceberg-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:498763380220a1d8881d52c218318e884be28c3ea5824cbbcb22a042c12a9ad3", size = 567243, upload-time = "2026-09-01T17:28:24.149Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bc/73277e56a30234afed4405bbe874a5410dbf9fa6c22c2352cc2615f7ed78/pyiceberg-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:307e46f46ffd48e0b270acf10bc5892f09e8c9fa2c828e9ffbf32ad480504bae", size = 567739, upload-time = "2026-09-01T17:28:25.376Z" }, + { url = "https://files.pythonhosted.org/packages/a2/72/8e09e90fd556af1ea90b993287da7997423a99784f4b7ccc77bd96a28f39/pyiceberg-0.12.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e055cc459d7b6eba21bd62eedbda0d0845ade0c2161a04c0f8eb252ec3e2d7d1", size = 773284, upload-time = "2026-09-01T17:28:26.452Z" }, + { url = "https://files.pythonhosted.org/packages/83/f1/cb542e8a46690d9cd2112eafd052f7bfdcff33f2027a344126d07b5b681a/pyiceberg-0.12.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:893e35df750644dab19bb873522d15914335e22368a711703176ea187c26b655", size = 776034, upload-time = "2026-09-01T17:28:27.823Z" }, + { url = "https://files.pythonhosted.org/packages/52/62/9e41c64c9bd741da75b408379ce3175af9f2dbf453c0a2bd9e5bd404e068/pyiceberg-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f53afc4ae649d35d43eee2b32515f22bc64fa0d4921c359ed9e7744ca1101ae0", size = 768107, upload-time = "2026-09-01T17:28:28.94Z" }, + { url = "https://files.pythonhosted.org/packages/9a/39/18af56141c920e62dcd4dc4f91aa058c7361e8f2e8dd45f73cf3f0c25b64/pyiceberg-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd57f73a55dc9439f183e13aa1d9c79ac4274564bdd412da735314e4e26b77ff", size = 774220, upload-time = "2026-09-01T17:28:30.094Z" }, + { url = "https://files.pythonhosted.org/packages/13/98/50a2a45e451df14ce0da864b1fd869a950c9cea0877800baa0ff287c5991/pyiceberg-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:934c30733c3debf9b13bbcdb85c4cfdaf4b72a808f7af1dd027a38e4e2baef07", size = 563797, upload-time = "2026-09-01T17:28:31.388Z" }, + { url = "https://files.pythonhosted.org/packages/19/e3/49ca88aff0dd74560acdcbf1d23d33a1e4e3a2d6d17f03399e7401faf084/pyiceberg-0.12.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7bb480f6ac06e4afb3de1e1a46781d071e3f1cfcc5802de9e48e6289b3546c0a", size = 567458, upload-time = "2026-09-01T17:28:32.72Z" }, + { url = "https://files.pythonhosted.org/packages/5e/90/18a84508ec3bae4e90b5f631a4f4ad7de477729b8fa90014c46550b8e1e6/pyiceberg-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:16f2f03c20d01ced43198aedfffb620163e7debc21d33ff02cc18e107aa43e69", size = 568050, upload-time = "2026-09-01T17:28:33.963Z" }, + { url = "https://files.pythonhosted.org/packages/54/a4/e3281a6e98645c179652c7a9c04bdc3fb2d388f26a75dbf3225891804a11/pyiceberg-0.12.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:115ecdabd4c47d64b4eda1a4271a4c4eea0514411832f11adc7cdf6630e57a66", size = 772948, upload-time = "2026-09-01T17:28:35.211Z" }, + { url = "https://files.pythonhosted.org/packages/70/d7/0b4fcd024b938dceeb941626ef465cc1d17d023ca461db58a636701b1482/pyiceberg-0.12.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb3db80cd510ddc34246059f4631661aa5b5d94c36b131499c56e746f22db3c5", size = 774232, upload-time = "2026-09-01T17:28:36.297Z" }, + { url = "https://files.pythonhosted.org/packages/97/bf/2c2d14235fcb1a28eba3e576e3472b77ee9408b52b13a2d33937a6189ea6/pyiceberg-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:82e0250f6c9baa11644efa44da5f968205c158c75126deb1b1730ce158cb3f76", size = 769045, upload-time = "2026-09-01T17:28:37.858Z" }, + { url = "https://files.pythonhosted.org/packages/ed/fb/c16e5aa6810a1e2dc0907b7a7d4605701b9da537d4175736cbec0c86ffb5/pyiceberg-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e73a396826d745b2a6ac0d3b5151c8cdcbb090e81bfc31f07139cb463e62dbe3", size = 772590, upload-time = "2026-09-01T17:28:39.132Z" }, + { url = "https://files.pythonhosted.org/packages/a4/57/2ae640b6220a321958484162cfbdd1a2772198bac0c4588a990b8afd22d5/pyiceberg-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:ee7b572d209a39224a093661f462f95551b2bd3982a07aecd80c7dd560be15ea", size = 561552, upload-time = "2026-09-01T17:28:41.156Z" }, ] [[package]] From fabb707075303d85f7e1e4b1b45834c3ec7cf68c Mon Sep 17 00:00:00 2001 From: Anurag Mantripragada Date: Mon, 14 Sep 2026 17:04:03 +0000 Subject: [PATCH 63/74] feat(iceberg): thread per-file sort order into FileScanTask (#3128) Resolves each data file's sort_order_id against the table's known sort orders during scan planning and carries the result on FileScanTask as sort_order (Some only when genuinely sorted; the reserved unsorted order, id 0, resolves to None) alongside the raw sort_order_id (kept unresolved so a non-zero id that fails to resolve is still recoverable as "physically sorted"). PlanContext precomputes a narrow sort-orders map once per scan, matching the existing unified_partition_type pattern, rather than carrying the full TableMetadataRef into every ManifestFileContext. --- crates/iceberg/public-api.txt | 4 +- crates/iceberg/src/scan/context.rs | 28 +++- crates/iceberg/src/scan/mod.rs | 205 ++++++++++++++++++++++++++++- crates/iceberg/src/scan/task.rs | 48 ++++++- 4 files changed, 276 insertions(+), 9 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index ca55804966..40987ef1df 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1292,6 +1292,8 @@ pub fn iceberg::scan::FileScanTask::project_field_ids(&self) -> &[i32] pub fn iceberg::scan::FileScanTask::record_count(&self) -> core::option::Option pub fn iceberg::scan::FileScanTask::schema(&self) -> &iceberg::spec::Schema pub fn iceberg::scan::FileScanTask::schema_ref(&self) -> iceberg::spec::SchemaRef +pub fn iceberg::scan::FileScanTask::sort_order(&self) -> core::option::Option<&iceberg::spec::SortOrderRef> +pub fn iceberg::scan::FileScanTask::sort_order_id(&self) -> core::option::Option pub fn iceberg::scan::FileScanTask::start(&self) -> u64 pub fn iceberg::scan::FileScanTask::unified_partition_type(&self) -> core::option::Option<&alloc::sync::Arc> impl core::clone::Clone for iceberg::scan::FileScanTask @@ -1304,7 +1306,7 @@ impl core::fmt::Debug for iceberg::scan::FileScanTask pub fn iceberg::scan::FileScanTask::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::scan::FileScanTask impl iceberg::scan::FileScanTask -pub fn iceberg::scan::FileScanTask::builder() -> FileScanTaskBuilder<((), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), ())> +pub fn iceberg::scan::FileScanTask::builder() -> FileScanTaskBuilder<((), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), ())> impl serde_core::ser::Serialize for iceberg::scan::FileScanTask pub fn iceberg::scan::FileScanTask::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::scan::FileScanTask diff --git a/crates/iceberg/src/scan/context.rs b/crates/iceberg/src/scan/context.rs index a598975b96..4635325fbf 100644 --- a/crates/iceberg/src/scan/context.rs +++ b/crates/iceberg/src/scan/context.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::sync::Arc; use futures::channel::mpsc::Sender; @@ -29,7 +30,7 @@ use crate::scan::{ }; use crate::spec::{ ManifestContentType, ManifestEntryRef, ManifestFile, ManifestList, NameMapping, - PartitionSpecRef, SchemaRef, SnapshotRef, StructType, TableMetadataRef, + PartitionSpecRef, SchemaRef, SnapshotRef, SortOrderRef, StructType, TableMetadataRef, }; use crate::{Error, ErrorKind, Result}; @@ -50,6 +51,7 @@ pub(crate) struct ManifestFileContext { case_sensitive: bool, partition_spec: Option, unified_partition_type: Option>, + sort_orders: Arc>, } /// Wraps a [`ManifestEntryRef`] alongside the objects that are needed @@ -67,6 +69,8 @@ pub(crate) struct ManifestEntryContext { pub case_sensitive: bool, pub partition_spec: Option, pub unified_partition_type: Option>, + pub sort_order_id: Option, + pub sort_order: Option, } impl ManifestFileContext { @@ -86,11 +90,23 @@ impl ManifestFileContext { case_sensitive, partition_spec, unified_partition_type, + sort_orders, } = self; let manifest = object_cache.get_manifest(&manifest_file).await?; for manifest_entry in manifest.entries() { + // Carry the raw id through unresolved, then resolve it to an order for the + // reader. The resolved order is `None` for the unsorted order (empty fields) so + // that `Some` always means genuinely sorted, matching Java's `isSorted()` gate; + // the raw id above keeps the "physically sorted but definition dropped" case + // recoverable. + let sort_order_id = manifest_entry.data_file().sort_order_id(); + let sort_order = sort_order_id + .and_then(|id| sort_orders.get(&(id as i64))) + .filter(|order| !order.is_unsorted()) + .cloned(); + let manifest_entry_context = ManifestEntryContext { // TODO: refactor to avoid the expensive ManifestEntry clone manifest_entry: manifest_entry.clone(), @@ -104,6 +120,8 @@ impl ManifestFileContext { case_sensitive, partition_spec: partition_spec.clone(), unified_partition_type: unified_partition_type.clone(), + sort_order_id, + sort_order, }; sender @@ -150,6 +168,8 @@ impl ManifestEntryContext { .with_unified_partition_type(self.unified_partition_type.clone()) .with_case_sensitive(self.case_sensitive) .with_key_metadata(self.manifest_entry.data_file.key_metadata().map(Box::from)) + .with_sort_order_id(self.sort_order_id) + .with_sort_order(self.sort_order) .build() } } @@ -174,6 +194,11 @@ pub(crate) struct PlanContext { pub expression_evaluator_cache: Arc, pub unified_partition_type: Option>, + + /// The table's sort orders keyed by id, precomputed once so each + /// [`ManifestFileContext`] carries only this narrow map rather than the full table + /// metadata. Mirrors how `unified_partition_type` carries a precomputed value. + pub sort_orders: Arc>, } impl PlanContext { @@ -304,6 +329,7 @@ impl PlanContext { .partition_spec_by_id(manifest_file.partition_spec_id) .cloned(), unified_partition_type: self.unified_partition_type.clone(), + sort_orders: self.sort_orders.clone(), } } } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index db70806473..9bdf42beb8 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -23,6 +23,7 @@ mod context; use context::*; mod task; +use std::collections::HashMap; use std::sync::Arc; use arrow_array::RecordBatch; @@ -41,7 +42,7 @@ use crate::metadata_columns::{ RESERVED_FIELD_ID_PARTITION, get_metadata_field_id, is_metadata_column_name, }; use crate::runtime::Runtime; -use crate::spec::{DataContentType, Schema, SchemaRef, SnapshotRef, StructType}; +use crate::spec::{DataContentType, Schema, SchemaRef, SnapshotRef, SortOrderRef, StructType}; use crate::table::Table; use crate::util::available_parallelism; use crate::{Error, ErrorKind, Result}; @@ -311,6 +312,16 @@ impl<'a> TableScanBuilder<'a> { .map(Arc::new); let unified_partition_type = projected_partition_type(self.table, &schema, &field_ids)?; + // Precompute the table's sort orders once, keyed by id, so each manifest-file + // context carries only this narrow map instead of the full table metadata. + let sort_orders = Arc::new( + self.table + .metadata() + .sort_orders_iter() + .map(|order| (order.order_id, order.clone())) + .collect::>(), + ); + let plan_context = PlanContext { snapshot, table_metadata: self.table.metadata_ref(), @@ -325,6 +336,7 @@ impl<'a> TableScanBuilder<'a> { manifest_evaluator_cache: Arc::new(ManifestEvaluatorCache::new()), expression_evaluator_cache: Arc::new(ExpressionEvaluatorCache::new()), unified_partition_type, + sort_orders, }; Ok(TableScan { @@ -664,9 +676,10 @@ pub mod tests { use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Datum, FormatVersion, Literal, MAIN_BRANCH, ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, - MappedField, NameMapping, NestedField, Operation, PartitionSpec, PrimitiveType, Schema, - Snapshot, Struct, StructType, Summary, TableMetadata, TableMetadataBuilder, - TableProperties, Transform, Type, UnboundPartitionSpec, + MappedField, NameMapping, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, + Schema, Snapshot, SortDirection, SortField, SortOrder, Struct, StructType, Summary, + TableMetadata, TableMetadataBuilder, TableProperties, Transform, Type, + UnboundPartitionSpec, }; use crate::table::Table; use crate::test_utils::test_runtime; @@ -1638,6 +1651,76 @@ pub mod tests { manifest_list_write.close().await.unwrap(); } + /// Writes a manifest with three live "Added" data-file entries (partitioned on `x` + /// = 100, 200, 300), each with the given `sort_order_id` set on its `DataFile` + /// (`None` leaves the field unset). Used to test how `sort_order_id` resolution + /// against the table's sort orders flows into each entry's `FileScanTask`. + pub async fn setup_manifest_files_with_sort_order_ids( + &mut self, + sort_order_ids: [Option; 4], + ) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + + for (i, sort_order_id) in sort_order_ids.into_iter().enumerate() { + let mut data_file_builder = DataFileBuilder::default(); + data_file_builder + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/{}.parquet", &self.table_location, i + 1)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long( + 100 * (i as i64 + 1), + ))])); + if let Some(id) = sort_order_id { + data_file_builder.sort_order_id(id); + } + let data_file = data_file_builder.build().unwrap(); + + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(data_file) + .build(), + ) + .unwrap(); + } + + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(std::iter::once(data_file_manifest)) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + /// Writes `mrg.parquet` with three 100-row row groups. Columns `x` (field /// id `1`) and `y` (field id `2`) both run 1000..1300, so row position `p` /// carries `x = y = 1000 + p`. Returns `(path, file_size_in_bytes)`. @@ -1969,6 +2052,104 @@ pub mod tests { } } + #[tokio::test] + async fn test_plan_files_carries_sort_order_into_file_scan_task() { + let mut fixture = TableTestFixture::new(); + + // Inject the reserved unsorted order (id 0) inline rather than editing the shared + // testdata fixture, so the id-0 file below exercises the `!is_unsorted()` filter + // branch instead of the `and_then` short-circuit an absent entry would take. + let mut metadata = fixture.table.metadata().clone(); + metadata + .sort_orders + .insert(0, Arc::new(SortOrder::unsorted_order())); + fixture.table = fixture.table.with_metadata(Arc::new(metadata)); + + let expected_sort_order = fixture + .table + .metadata() + .sort_order_by_id(3) + .unwrap() + .clone(); + + // sort_order_ids: resolvable (3), absent, unresolvable (99), reserved unsorted (0). + fixture + .setup_manifest_files_with_sort_order_ids([Some(3), None, Some(99), Some(0)]) + .await; + + let tasks: Vec<_> = fixture + .table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + assert_eq!(tasks.len(), 4, "expected all four FileScanTasks"); + + // Aggregates catch a systemic regression (every entry resolving to id 3, or + // resolution dropping entirely) that the per-file checks below would each still pass. + assert_eq!( + tasks.iter().filter(|t| t.sort_order().is_some()).count(), + 1, + "exactly one file resolves to a sort order" + ); + assert_eq!( + tasks.iter().filter(|t| t.sort_order_id().is_some()).count(), + 3, + "three files carry a raw sort_order_id" + ); + + let resolved = tasks + .iter() + .find(|t| t.data_file_path().ends_with("1.parquet")) + .unwrap(); + assert_eq!(resolved.sort_order_id(), Some(3)); + assert_eq!( + resolved.sort_order(), + Some(&expected_sort_order), + "sort_order_id 3 should resolve to the table's sort order at id 3" + ); + + let missing = tasks + .iter() + .find(|t| t.data_file_path().ends_with("2.parquet")) + .unwrap(); + assert_eq!(missing.sort_order_id(), None); + assert!( + missing.sort_order().is_none(), + "a file with no sort_order_id carries no sort_order" + ); + + let unresolvable = tasks + .iter() + .find(|t| t.data_file_path().ends_with("3.parquet")) + .unwrap(); + assert_eq!( + unresolvable.sort_order_id(), + Some(99), + "the raw id is preserved even when it does not resolve" + ); + assert!( + unresolvable.sort_order().is_none(), + "an unresolvable sort_order_id resolves to no sort_order" + ); + + let unsorted = tasks + .iter() + .find(|t| t.data_file_path().ends_with("4.parquet")) + .unwrap(); + assert_eq!(unsorted.sort_order_id(), Some(0)); + assert!( + unsorted.sort_order().is_none(), + "the reserved unsorted order (id 0) resolves to no sort_order" + ); + } + #[tokio::test] async fn test_plan_files_on_table_without_any_snapshots() { let table = TableTestFixture::new_empty().table; @@ -2746,6 +2927,20 @@ pub mod tests { .unwrap(), ); let unified_partition_type = Arc::new(partition_spec.partition_type(&schema).unwrap()); + let sort_order = Arc::new( + SortOrder::builder() + .with_order_id(1) + .with_sort_field( + SortField::builder() + .source_id(1) + .transform(Transform::Identity) + .direction(SortDirection::Ascending) + .null_order(NullOrder::First) + .build(), + ) + .build(&schema) + .unwrap(), + ); let task = FileScanTask::builder() .with_data_file_path("data_file_path".to_string()) .with_file_size_in_bytes(123) @@ -2777,6 +2972,8 @@ pub mod tests { vec![], )])))) .with_unified_partition_type(Some(unified_partition_type)) + .with_sort_order_id(Some(1)) + .with_sort_order(Some(sort_order)) .with_case_sensitive(true) .with_key_metadata(Some(vec![1, 2, 3].into_boxed_slice())) .build() diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index f1a799fcc6..5fd0f81bb6 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -24,7 +24,7 @@ use typed_builder::TypedBuilder; use crate::expr::BoundPredicate; use crate::spec::{ DataContentType, DataFileFormat, ManifestEntryRef, NameMapping, PartitionSpec, Schema, - SchemaRef, Struct, StructType, + SchemaRef, SortOrderRef, Struct, StructType, }; use crate::{Error, ErrorKind, Result}; @@ -116,6 +116,26 @@ pub struct FileScanTask { #[builder(default)] unified_partition_type: Option>, + /// The raw `sort_order_id` recorded on the data file (spec field 140), carried through + /// unresolved. See [`DataFile::sort_order_id()`](crate::spec::DataFile::sort_order_id). + /// `None` if the file has no sort order id. + /// + /// This preserves the distinction that [`sort_order`](Self::sort_order) collapses: a + /// non-zero id that does not resolve against the table's sort orders still marks the file + /// as physically sorted, which a consumer such as the DataFusion sorted-scan optimizer + /// needs even when the order definition itself is unavailable. + #[builder(default)] + sort_order_id: Option, + + /// The sort order that this file's rows are sorted by, resolved from the data file's + /// [`sort_order_id`](Self::sort_order_id) against the table's known sort orders. `Some` + /// only when the id resolves to an order that has sort fields. `None` if the file has no + /// sort order id, the id does not resolve, or it resolves to an order with no sort fields + /// (the reserved unsorted order, id 0 per the spec). Use [`sort_order_id`](Self::sort_order_id) + /// to tell those cases apart. + #[builder(default)] + sort_order: Option, + /// Whether this scan task should treat column names as case-sensitive when binding predicates. case_sensitive: bool, @@ -217,6 +237,16 @@ impl FileScanTask { self.unified_partition_type.as_ref() } + /// Returns the raw sort order id recorded on the data file, unresolved. + pub fn sort_order_id(&self) -> Option { + self.sort_order_id + } + + /// Returns the sort order this file's rows are sorted by, if resolved. + pub fn sort_order(&self) -> Option<&SortOrderRef> { + self.sort_order.as_ref() + } + /// Returns whether names are treated as case-sensitive. pub fn case_sensitive(&self) -> bool { self.case_sensitive @@ -372,8 +402,8 @@ mod _serde { use super::{FileScanTask, FileScanTaskDeleteFile}; use crate::expr::BoundPredicate; use crate::spec::{ - DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral, SchemaRef, StructType, - Type, + DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral, SchemaRef, SortOrderRef, + StructType, Type, }; use crate::{Error, ErrorKind, Result}; @@ -399,6 +429,10 @@ mod _serde { name_mapping: Option>, #[serde(default)] unified_partition_type: Option>, + #[serde(default)] + sort_order_id: Option, + #[serde(default)] + sort_order: Option, case_sensitive: bool, #[serde(default)] key_metadata: Option>, @@ -430,6 +464,10 @@ mod _serde { name_mapping: Option<&'a Arc>, #[serde(skip_serializing_if = "Option::is_none")] unified_partition_type: Option<&'a Arc>, + #[serde(skip_serializing_if = "Option::is_none")] + sort_order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + sort_order: Option<&'a SortOrderRef>, case_sensitive: bool, #[serde(skip_serializing_if = "Option::is_none")] key_metadata: Option<&'a [u8]>, @@ -477,6 +515,8 @@ mod _serde { partition_spec: value.partition_spec.as_ref(), name_mapping: value.name_mapping.as_ref(), unified_partition_type: value.unified_partition_type.as_ref(), + sort_order_id: value.sort_order_id, + sort_order: value.sort_order.as_ref(), case_sensitive: value.case_sensitive, key_metadata: value.key_metadata.as_deref(), }) @@ -528,6 +568,8 @@ mod _serde { .with_partition_spec(value.partition_spec) .with_name_mapping(value.name_mapping) .with_unified_partition_type(value.unified_partition_type) + .with_sort_order_id(value.sort_order_id) + .with_sort_order(value.sort_order) .with_case_sensitive(value.case_sensitive) .with_key_metadata(value.key_metadata) .build() From f11a20d0ba9b6c168839e234f3817b0e0d75ce4b Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 14 Sep 2026 17:58:23 +0000 Subject: [PATCH 64/74] perf(arrow): build partition filter mask with BooleanBufferBuilder (#3204) The partition splitter built each group's selection mask as a throwaway `vec![false; num_rows]` and bit-packed it into a `BooleanArray` via `.into()`, allocating an n-byte `Vec` plus a full packing pass per group per batch on every partitioned write. Build the packed buffer directly with `BooleanBufferBuilder`, setting only the selected bits. --- .../src/arrow/record_batch_partition_splitter.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs index c773947c7b..598bbf09f7 100644 --- a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs +++ b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; use std::sync::Arc; use arrow_array::{ArrayRef, BooleanArray, RecordBatch, StructArray}; +use arrow_buffer::BooleanBufferBuilder; use arrow_select::filter::filter_record_batch; use super::arrow_struct_to_literal; @@ -190,11 +191,12 @@ impl RecordBatchPartitionSplitter { for (row, row_ids) in group_ids.into_iter() { // generate the bool filter array from column_ids let filter_array: BooleanArray = { - let mut filter = vec![false; batch.num_rows()]; - row_ids.into_iter().for_each(|row_id| { - filter[row_id] = true; - }); - filter.into() + let mut builder = BooleanBufferBuilder::new(batch.num_rows()); + builder.append_n(batch.num_rows(), false); + for row_id in row_ids { + builder.set_bit(row_id, true); + } + BooleanArray::new(builder.finish(), None) }; // Create PartitionKey from the partition struct From f0c9dad705f27f8745a7b8ad92a9768ce1d9548c Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 14 Sep 2026 18:00:20 +0000 Subject: [PATCH 65/74] perf(expr): borrow field type in manifest bound decoding instead of cloning (#3214) `ManifestFilterVisitor::bytes_to_datum` took the field `Type` by value, so every comparison-leaf call site cloned the field's boxed `Type` to decode a bound -- and the two in-predicate sites cloned the entire `NestedField`. `bytes_to_datum` only borrows the type to reach its `PrimitiveType`, so take `&Type` and pass `&reference.field().field_type` at the call sites, dropping the per-leaf clones on the manifest-pruning path. --- .../src/expr/visitors/manifest_evaluator.rs | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/crates/iceberg/src/expr/visitors/manifest_evaluator.rs b/crates/iceberg/src/expr/visitors/manifest_evaluator.rs index 53dbafc065..61f01cd5e8 100644 --- a/crates/iceberg/src/expr/visitors/manifest_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/manifest_evaluator.rs @@ -189,7 +189,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { Some(bound_bytes) => { let bound = ManifestFilterVisitor::bytes_to_datum( bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if datum <= &bound { ROWS_CANNOT_MATCH @@ -212,7 +212,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { Some(bound_bytes) => { let bound = ManifestFilterVisitor::bytes_to_datum( bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if datum < &bound { ROWS_CANNOT_MATCH @@ -235,7 +235,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { Some(bound_bytes) => { let bound = ManifestFilterVisitor::bytes_to_datum( bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if datum >= &bound { ROWS_CANNOT_MATCH @@ -258,7 +258,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { Some(bound_bytes) => { let bound = ManifestFilterVisitor::bytes_to_datum( bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if datum > &bound { ROWS_CANNOT_MATCH @@ -285,7 +285,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { if let Some(lower_bound_bytes) = &field.lower_bound { let lower_bound = ManifestFilterVisitor::bytes_to_datum( lower_bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if &lower_bound > datum { return ROWS_CANNOT_MATCH; @@ -295,7 +295,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { if let Some(upper_bound_bytes) = &field.upper_bound { let upper_bound = ManifestFilterVisitor::bytes_to_datum( upper_bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if &upper_bound < datum { return ROWS_CANNOT_MATCH; @@ -410,20 +410,16 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { } if let Some(lower_bound) = &field.lower_bound { - let lower_bound = ManifestFilterVisitor::bytes_to_datum( - lower_bound, - *reference.field().clone().field_type, - ); + let lower_bound = + ManifestFilterVisitor::bytes_to_datum(lower_bound, &reference.field().field_type); if literals.iter().all(|datum| &lower_bound > datum) { return ROWS_CANNOT_MATCH; } } if let Some(upper_bound) = &field.upper_bound { - let upper_bound = ManifestFilterVisitor::bytes_to_datum( - upper_bound, - *reference.field().clone().field_type, - ); + let upper_bound = + ManifestFilterVisitor::bytes_to_datum(upper_bound, &reference.field().field_type); if literals.iter().all(|datum| &upper_bound < datum) { return ROWS_CANNOT_MATCH; } @@ -474,7 +470,7 @@ impl ManifestFilterVisitor<'_> { Ok(bound) } - fn bytes_to_datum(bytes: &ByteBuf, t: Type) -> Datum { + fn bytes_to_datum(bytes: &ByteBuf, t: &Type) -> Datum { let p = t.as_primitive_type().unwrap(); Datum::try_from_bytes(bytes, p.clone()).unwrap() } From 9db0b68572da283fcef96d7d4340e3c1ac4cc2ec Mon Sep 17 00:00:00 2001 From: Andrei Tserakhau Date: Mon, 14 Sep 2026 21:10:52 +0000 Subject: [PATCH 66/74] feat(writer): add PositionDeleteFileWriter (#2986) * feat(writer): add PositionDeleteFileWriter Add PositionDeleteFileWriter and its builder under writer/base_writer. It writes a position delete file with the two required columns file_path (string, field id 2147483546) and pos (long, 2147483545) and sets DataContentType::PositionDeletes on close. position_delete_schema() returns the canonical Iceberg schema, built from the existing metadata_columns field definitions. write() checks that a batch has exactly those two required, non-nullable, correctly typed columns before writing, and rejects a closed writer. close() propagates the partition key and leaves sort_order_id null. Position delete files are a v2 construct: v3 replaces them with deletion vectors and forbids adding new ones, so callers must not route v3 writes here. This base writer has no format-version gate by design; that gating belongs at the transaction/commit layer. The writer also does not sort its input, so callers must supply rows sorted by (file_path, pos) until a higher-level writer enforces it. Setting referenced_data_file and a higher-level DeltaWriter are follow-ups. Tested: schema shape, a parquet round trip for single and multiple writes, partition propagation, that the reserved field ids survive into the written parquet schema, and the validation cases (wrong column count, wrong/missing/ unparsable field ids, wrong types including LargeUtf8, nullable columns, and writes after close). Refs #340. * docs(writer): note the optional row column is unsupported The position delete spec allows an optional `row` column alongside `file_path` and `pos` that inlines the deleted row's values. This writer handles only the two required columns, so say so in the module docs. The batch validator already rejects a third column with a matching message. --------- Co-authored-by: Shawn Chang --- crates/iceberg/public-api.txt | 22 + crates/iceberg/src/writer/base_writer/mod.rs | 1 + .../base_writer/position_delete_writer.rs | 916 ++++++++++++++++++ 3 files changed, 939 insertions(+) create mode 100644 crates/iceberg/src/writer/base_writer/position_delete_writer.rs diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 40987ef1df..9ba84036b7 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3342,6 +3342,22 @@ pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWrite pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig::projected_arrow_schema_ref(&self) -> &arrow_schema::schema::SchemaRef impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub mod iceberg::writer::base_writer::position_delete_writer +pub struct iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder +impl iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder) -> Self +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::R = iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn iceberg::writer::base_writer::position_delete_writer::position_delete_schema() -> iceberg::spec::SchemaRef pub mod iceberg::writer::file_writer pub mod iceberg::writer::file_writer::location_generator pub struct iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator @@ -3491,6 +3507,9 @@ pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter:: impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub trait iceberg::writer::IcebergWriterBuilder: core::marker::Send + core::marker::Sync + 'static pub type iceberg::writer::IcebergWriterBuilder::R: iceberg::writer::IcebergWriter pub fn iceberg::writer::IcebergWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait @@ -3500,6 +3519,9 @@ pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator pub type iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::R = iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::R = iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait pub macro iceberg::ensure_data_valid! #[non_exhaustive] pub enum iceberg::ErrorKind pub iceberg::ErrorKind::CatalogCommitConflicts diff --git a/crates/iceberg/src/writer/base_writer/mod.rs b/crates/iceberg/src/writer/base_writer/mod.rs index 37ab97eb6d..5a23931f25 100644 --- a/crates/iceberg/src/writer/base_writer/mod.rs +++ b/crates/iceberg/src/writer/base_writer/mod.rs @@ -19,3 +19,4 @@ pub mod data_file_writer; pub mod equality_delete_writer; +pub mod position_delete_writer; diff --git a/crates/iceberg/src/writer/base_writer/position_delete_writer.rs b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs new file mode 100644 index 0000000000..ebe3e95882 --- /dev/null +++ b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs @@ -0,0 +1,916 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module provides `PositionDeleteFileWriter`. +//! +//! A position delete file has two required columns: `file_path` (`string`, field id +//! [`RESERVED_FIELD_ID_DELETE_FILE_PATH`]) and `pos` (`long`, field id +//! [`RESERVED_FIELD_ID_DELETE_FILE_POS`]). The spec also allows an optional `row` column +//! that inlines the deleted row's values; this writer does not support it yet, so batches +//! must contain only the two required columns. The writer takes batches already shaped as +//! those two columns (see [`position_delete_schema`]) and sets +//! [`DataContentType::PositionDeletes`] on the output. It does not sort its input; see +//! [`PositionDeleteFileWriter::write`]. +//! +//! Position delete files are a v2 construct. v3 replaces them with deletion vectors and +//! forbids adding new position delete files, so callers must not route v3 writes here. +//! This base writer has no format-version gate by design; that gating belongs at the +//! transaction/commit layer. + +use std::sync::Arc; + +use arrow_array::RecordBatch; +use arrow_schema::{DataType, Field}; +use once_cell::sync::Lazy; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + +use crate::metadata_columns::{ + RESERVED_FIELD_ID_DELETE_FILE_PATH, RESERVED_FIELD_ID_DELETE_FILE_POS, delete_file_path_field, + delete_file_pos_field, +}; +use crate::spec::{DataContentType, DataFile, PartitionKey, Schema, SchemaRef}; +use crate::writer::file_writer::FileWriterBuilder; +use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; +use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder}; +use crate::writer::{IcebergWriter, IcebergWriterBuilder}; +use crate::{Error, ErrorKind, Result}; + +/// The canonical Iceberg schema of a position delete file: the required `file_path` +/// (`string`) and `pos` (`long`) columns with their reserved field ids. +static POSITION_DELETE_SCHEMA: Lazy = Lazy::new(|| { + Arc::new( + Schema::builder() + .with_fields(vec![ + delete_file_path_field().clone(), + delete_file_pos_field().clone(), + ]) + .build() + .expect("position delete schema is statically valid"), + ) +}); + +/// [`POSITION_DELETE_SCHEMA`] converted to Arrow, keeping the reserved field ids in each +/// field's Parquet field-id metadata. Test-only for now: callers configure the writer with +/// the Iceberg [`position_delete_schema`], so no non-test code needs the Arrow form yet. +#[cfg(test)] +static POSITION_DELETE_ARROW_SCHEMA: Lazy = Lazy::new(|| { + Arc::new( + crate::arrow::schema_to_arrow_schema(&POSITION_DELETE_SCHEMA) + .expect("position delete arrow schema is statically valid"), + ) +}); + +/// Returns the canonical Iceberg schema of a position delete file. +/// +/// Use this to build the [`ParquetWriterBuilder`](crate::writer::file_writer::ParquetWriterBuilder) +/// that backs a [`PositionDeleteFileWriter`], so the written file matches the +/// spec exactly. +pub fn position_delete_schema() -> SchemaRef { + POSITION_DELETE_SCHEMA.clone() +} + +/// Returns the canonical Arrow schema of a position delete file. +#[cfg(test)] +fn position_delete_arrow_schema() -> arrow_schema::SchemaRef { + POSITION_DELETE_ARROW_SCHEMA.clone() +} + +/// Reads a field's Iceberg field id from its Parquet field-id metadata. +fn field_id(field: &Field) -> Result { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Position delete column `{}` is missing its Iceberg field id metadata.", + field.name() + ), + ) + })? + .parse::() + .map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Position delete column `{}` has an invalid field id: {e}", + field.name() + ), + ) + }) +} + +/// Validates that a batch is a position delete file: the `file_path` (`Utf8`) and +/// `pos` (`Int64`) columns, in order, with the two reserved field ids. Checking it +/// here gives a clear error before the batch reaches the Parquet writer. +fn validate_position_delete_batch(batch: &RecordBatch) -> Result<()> { + let fields = batch.schema_ref().fields(); + if fields.len() != 2 { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "This writer supports only the two required position delete columns (`file_path`, `pos`); \ + batches with a different column count (e.g. including the optional `row` column) are not supported. Got {} columns.", + fields.len() + ), + )); + } + + let path = &fields[0]; + let path_id = field_id(path)?; + if path_id != RESERVED_FIELD_ID_DELETE_FILE_PATH { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "The first position delete column must be `file_path` (field id {RESERVED_FIELD_ID_DELETE_FILE_PATH}), but got field id {path_id}." + ), + )); + } + // The canonical schema maps Iceberg `string` to `Utf8` and the file writer is + // configured with it, so a `LargeUtf8` column has to be cast to `Utf8` first. + if path.data_type() != &DataType::Utf8 { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "The position delete `file_path` column must be Utf8 (cast it first); got {:?}.", + path.data_type() + ), + )); + } + // Required column: a nullable field could write nulls under a required schema. + if path.is_nullable() { + return Err(Error::new( + ErrorKind::DataInvalid, + "The position delete `file_path` column must be required (non-nullable).", + )); + } + + let pos = &fields[1]; + let pos_id = field_id(pos)?; + if pos_id != RESERVED_FIELD_ID_DELETE_FILE_POS { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "The second position delete column must be `pos` (field id {RESERVED_FIELD_ID_DELETE_FILE_POS}), but got field id {pos_id}." + ), + )); + } + if pos.data_type() != &DataType::Int64 { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "The position delete `pos` column must be Int64, but got {:?}.", + pos.data_type() + ), + )); + } + if pos.is_nullable() { + return Err(Error::new( + ErrorKind::DataInvalid, + "The position delete `pos` column must be required (non-nullable).", + )); + } + + Ok(()) +} + +/// Builder for [`PositionDeleteFileWriter`]. +#[derive(Debug)] +pub struct PositionDeleteFileWriterBuilder< + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +> { + inner: RollingFileWriterBuilder, +} + +impl PositionDeleteFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + /// Create a new `PositionDeleteFileWriterBuilder` using a `RollingFileWriterBuilder`. + /// + /// The `RollingFileWriterBuilder` must be backed by a file writer configured + /// with the [`position_delete_schema`]; the per-batch validation in + /// [`PositionDeleteFileWriter::write`] guards against a mismatched batch, but + /// the caller is responsible for wiring the same schema into the file writer. + pub fn new(inner: RollingFileWriterBuilder) -> Self { + Self { inner } + } +} + +#[async_trait::async_trait] +impl IcebergWriterBuilder for PositionDeleteFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + type R = PositionDeleteFileWriter; + + async fn build(&self, partition_key: Option) -> Result { + Ok(PositionDeleteFileWriter { + inner: Some(self.inner.build()), + partition_key, + }) + } +} + +/// Writer used to write position delete files within one spec/partition. +#[derive(Debug)] +pub struct PositionDeleteFileWriter< + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +> { + inner: Option>, + partition_key: Option, +} + +#[async_trait::async_trait] +impl IcebergWriter for PositionDeleteFileWriter +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + /// Writes a batch of `(file_path, pos)` records; the shape is validated on every + /// call. + /// + /// The writer does not sort its input. Position delete files must be sorted by + /// `file_path` then `pos`, so the caller must supply rows in that order across all + /// `write` calls; a sorting writer will remove this requirement. + async fn write(&mut self, batch: RecordBatch) -> Result<()> { + // Reject a closed writer before validating the batch. + let Some(writer) = self.inner.as_mut() else { + return Err(Error::new( + ErrorKind::Unexpected, + "Position delete writer is already closed; cannot write.", + )); + }; + validate_position_delete_batch(&batch)?; + writer.write(&self.partition_key, &batch).await + } + + async fn close(&mut self) -> Result> { + if let Some(writer) = self.inner.take() { + writer + .close() + .await? + .into_iter() + .map(|mut res| { + res.content(DataContentType::PositionDeletes); + // sort_order_id stays null, as the spec requires for position deletes. + if let Some(pk) = self.partition_key.as_ref() { + res.partition(pk.data().clone()); + res.partition_spec_id(pk.spec().spec_id()); + } + res.build().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Failed to build position delete file: {e}"), + ) + }) + }) + .collect() + } else { + Err(Error::new( + ErrorKind::Unexpected, + "Position delete writer is already closed.", + )) + } + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{Int32Array, Int64Array, LargeStringArray, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field}; + use arrow_select::concat::concat_batches; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use parquet::file::properties::WriterProperties; + use tempfile::TempDir; + + use super::*; + use crate::io::FileIO; + use crate::metadata_columns::{ + RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS, + }; + use crate::spec::{ + DataFileFormat, Literal, NestedField, PartitionSpec, PrimitiveType, Struct, Transform, Type, + }; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + + #[test] + fn test_position_delete_schema_shape() { + let schema = position_delete_schema(); + let fields = schema.as_struct().fields(); + assert_eq!(fields.len(), 2); + + assert_eq!(fields[0].id, RESERVED_FIELD_ID_DELETE_FILE_PATH); + assert_eq!(fields[0].name, RESERVED_COL_NAME_DELETE_FILE_PATH); + assert!(fields[0].required); + assert_eq!( + fields[0].field_type.as_ref(), + &Type::Primitive(PrimitiveType::String) + ); + + assert_eq!(fields[1].id, RESERVED_FIELD_ID_DELETE_FILE_POS); + assert_eq!(fields[1].name, RESERVED_COL_NAME_DELETE_FILE_POS); + assert!(fields[1].required); + assert_eq!( + fields[1].field_type.as_ref(), + &Type::Primitive(PrimitiveType::Long) + ); + + // The Arrow projection carries the reserved field ids and non-null flags. + let arrow_schema = position_delete_arrow_schema(); + assert_eq!(arrow_schema.fields().len(), 2); + assert_eq!(arrow_schema.field(0).data_type(), &DataType::Utf8); + assert_eq!(arrow_schema.field(1).data_type(), &DataType::Int64); + assert!(!arrow_schema.field(0).is_nullable()); + assert!(!arrow_schema.field(1).is_nullable()); + } + + fn position_delete_batch(paths: Vec<&str>, positions: Vec) -> RecordBatch { + RecordBatch::try_new(position_delete_arrow_schema(), vec![ + Arc::new(StringArray::from(paths)), + Arc::new(Int64Array::from(positions)), + ]) + .unwrap() + } + + /// A field carrying an explicit Iceberg field-id metadata entry. + fn field_with_id(name: &str, data_type: DataType, field_id: i32) -> Field { + Field::new(name, data_type, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + field_id.to_string(), + )])) + } + + fn writer_setup( + temp_dir: &TempDir, + ) -> ( + FileIO, + PositionDeleteFileWriterBuilder< + ParquetWriterBuilder, + DefaultLocationGenerator, + DefaultFileNameGenerator, + >, + ) { + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let parquet_writer_builder = ParquetWriterBuilder::new( + WriterProperties::builder().build(), + position_delete_schema(), + ); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + ( + file_io, + PositionDeleteFileWriterBuilder::new(rolling_writer_builder), + ) + } + + #[tokio::test] + async fn test_position_delete_writer_round_trip() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Sorted by (file_path, pos): f0/1, f0/4, then f1/2. + let batch = position_delete_batch( + vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f1.parquet", + ], + vec![1, 4, 2], + ); + writer.write(batch.clone()).await?; + let data_files = writer.close().await?; + + assert_eq!(data_files.len(), 1); + let data_file = &data_files[0]; + assert_eq!(data_file.content_type(), DataContentType::PositionDeletes); + assert_eq!(data_file.file_format, DataFileFormat::Parquet); + assert_eq!(data_file.record_count, 3); + // Unpartitioned writer leaves the default (empty) partition / spec id. + assert_eq!(data_file.partition, Struct::empty()); + assert_eq!(data_file.partition_spec_id, 0); + // The rolling writer fills in file statistics. + assert!(data_file.file_size_in_bytes > 0); + + // The written Parquet file round-trips back to the exact input rows. + let read_back = read_back_single(&file_io, data_file, &batch.schema()).await; + assert_eq!(read_back, batch); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_parquet_field_ids() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + writer + .write(position_delete_batch( + vec!["s3://bucket/data/f0.parquet"], + vec![1], + )) + .await?; + let data_files = writer.close().await?; + + // The reserved field ids must survive into the written Parquet schema, not just + // the in-memory Arrow schema, so cross-engine readers resolve the columns by id. + let content = file_io + .new_input(data_files[0].file_path.clone())? + .read() + .await?; + let reader = ParquetRecordBatchReaderBuilder::try_new(content).unwrap(); + let field_ids: Vec = reader + .parquet_schema() + .columns() + .iter() + .map(|col| col.self_type().get_basic_info().id()) + .collect(); + assert_eq!(field_ids, vec![ + RESERVED_FIELD_ID_DELETE_FILE_PATH, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ]); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_multiple_writes() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + let batch1 = position_delete_batch( + vec!["s3://bucket/data/f0.parquet", "s3://bucket/data/f0.parquet"], + vec![1, 4], + ); + let batch2 = position_delete_batch( + vec!["s3://bucket/data/f1.parquet", "s3://bucket/data/f1.parquet"], + vec![2, 7], + ); + writer.write(batch1.clone()).await?; + writer.write(batch2.clone()).await?; + let data_files = writer.close().await?; + + assert_eq!(data_files.len(), 1); + assert_eq!(data_files[0].record_count, 4); + + let expected = concat_batches(&batch1.schema(), [&batch1, &batch2]).unwrap(); + let read_back = read_back_single(&file_io, &data_files[0], &batch1.schema()).await; + assert_eq!(read_back, expected); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_sets_partition() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + + // A table schema + identity partition spec with a non-default spec id, so the + // assertions distinguish real propagation from the DataFileBuilder defaults. + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?, + ); + let spec = PartitionSpec::builder(table_schema.clone()) + .with_spec_id(7) + .add_partition_field("id", "id", Transform::Identity)? + .build()?; + let partition_value = Struct::from_iter([Some(Literal::int(42))]); + let partition_key = PartitionKey::new(spec, table_schema.clone(), partition_value.clone()); + + let mut writer = builder.build(Some(partition_key)).await?; + writer + .write(position_delete_batch( + vec!["s3://bucket/data/f0.parquet"], + vec![1], + )) + .await?; + let data_files = writer.close().await?; + + assert_eq!(data_files.len(), 1); + let data_file = &data_files[0]; + assert_eq!(data_file.content_type(), DataContentType::PositionDeletes); + assert_eq!(data_file.partition_spec_id, 7); + assert_eq!(data_file.partition, partition_value); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_wrong_column_count() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // A batch carrying only the `file_path` column is not a position delete file. + let arrow_schema = position_delete_arrow_schema(); + let path_only = RecordBatch::try_new(Arc::new(arrow_schema.project(&[0]).unwrap()), vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + ]) + .unwrap(); + + let err = writer.write(path_only).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("only the two required position delete columns"), + "{err}" + ); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_missing_field_ids() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Correct shape and types, but plain field names without the reserved + // Iceberg field-id metadata: must be rejected. + let plain_schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new(RESERVED_COL_NAME_DELETE_FILE_PATH, DataType::Utf8, false), + Field::new(RESERVED_COL_NAME_DELETE_FILE_POS, DataType::Int64, false), + ])); + let batch = RecordBatch::try_new(plain_schema, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("field id metadata"), "{err}"); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_unparsable_field_id() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Field-id metadata present but not an integer: must be rejected. + let bad_meta = Arc::new(arrow_schema::Schema::new(vec![ + Field::new(RESERVED_COL_NAME_DELETE_FILE_PATH, DataType::Utf8, false).with_metadata( + HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "not_a_number".to_string(), + )]), + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(bad_meta, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("invalid field id"), "{err}"); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_wrong_field_id() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Right shape and types, but the two reserved field ids are swapped. + let swapped = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + ])); + let batch = RecordBatch::try_new(swapped, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be `file_path`"), "{err}"); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_bad_pos_field_id() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Correct `file_path`, but `pos` carries the wrong reserved field id — the + // pos-specific branch (not the path branch) must fire. + let wrong_pos_id = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id(RESERVED_COL_NAME_DELETE_FILE_POS, DataType::Int64, 999), + ])); + let batch = RecordBatch::try_new(wrong_pos_id, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be `pos`"), "{err}"); + + // Correct `file_path`, but `pos` is missing its field-id metadata entirely. + let missing_pos_id = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + Field::new(RESERVED_COL_NAME_DELETE_FILE_POS, DataType::Int64, false), + ])); + let batch = RecordBatch::try_new(missing_pos_id, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("field id metadata"), "{err}"); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_wrong_column_types() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Correct field ids, but `pos` is Int32 rather than Int64. + let int32_pos = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int32, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(int32_pos, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int32Array::from(vec![1_i32])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be Int64"), "{err}"); + + // Correct field ids, but `file_path` is not a string. + let int_path = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Int32, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(int_path, vec![ + Arc::new(Int32Array::from(vec![1_i32])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be Utf8"), "{err}"); + + // Correct field ids, but `file_path` is LargeUtf8 — the common shape from + // DataFusion/DuckDB/Polars, and the case the validator's comment calls out. + let large_path = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::LargeUtf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(large_path, vec![ + Arc::new(LargeStringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be Utf8"), "{err}"); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_nullable_columns() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Correct field ids and types, but the columns are declared nullable. A + // required schema with a nullable field could emit nulls -> malformed file. + let nullable = |name: &str, data_type: DataType, id: i32| { + Field::new(name, data_type, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + id.to_string(), + )])) + }; + + let nullable_path = Arc::new(arrow_schema::Schema::new(vec![ + nullable( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(nullable_path, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("`file_path` column must be required"), + "{err}" + ); + + let nullable_pos = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + nullable( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(nullable_pos, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains("`pos` column must be required"), + "{err}" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_close_without_writes() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Closing a writer that never received a batch produces no data files. + let data_files = writer.close().await?; + assert!(data_files.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_errors_after_close() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + writer.close().await?; + + // Both write() and a second close() report the writer is already closed. + let write_err = writer + .write(position_delete_batch( + vec!["s3://bucket/data/f0.parquet"], + vec![1], + )) + .await + .unwrap_err(); + assert_eq!(write_err.kind(), ErrorKind::Unexpected); + assert!( + write_err.to_string().contains("cannot write"), + "{write_err}" + ); + + // Even a malformed batch surfaces the closed error, not a validation error: + // the closed check runs before validation. + let arrow_schema = position_delete_arrow_schema(); + let invalid = RecordBatch::try_new(Arc::new(arrow_schema.project(&[0]).unwrap()), vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + ]) + .unwrap(); + let invalid_err = writer.write(invalid).await.unwrap_err(); + assert_eq!(invalid_err.kind(), ErrorKind::Unexpected); + + let close_err = writer.close().await.unwrap_err(); + assert_eq!(close_err.kind(), ErrorKind::Unexpected); + assert!( + close_err.to_string().contains("already closed"), + "{close_err}" + ); + Ok(()) + } + + async fn read_back_single( + file_io: &FileIO, + data_file: &DataFile, + schema: &arrow_schema::SchemaRef, + ) -> RecordBatch { + let input_content = file_io + .new_input(data_file.file_path.clone()) + .unwrap() + .read() + .await + .unwrap(); + let reader = ParquetRecordBatchReaderBuilder::try_new(input_content) + .unwrap() + .build() + .unwrap(); + let batches = reader.map(|b| b.unwrap()).collect::>(); + concat_batches(schema, &batches).unwrap() + } +} From 5c3d18f4ad22656f7aa13c03657190a0fd5a7d37 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Tue, 15 Sep 2026 07:43:57 +0000 Subject: [PATCH 67/74] fix(release): warn when bash 3.2 will not report the failed step (#3194) The release scripts run each step in a `( trap - ERR; ... )` subshell so a failure is reported exactly once. bash 3.2 (macOS /bin/bash) does not run the parent's ERR trap for a failing subshell, so the script still exits on failure but does not report which step failed. bash 4.0 and later do; verified against 3.2 through 5.1. Print a warning at startup on bash older than 4 in the four scripts that install the trap. Co-authored-by: Claude Fable 5.1 --- dev/release/create_rc.sh | 6 ++++++ dev/release/dependencies.sh | 6 ++++++ dev/release/release.sh | 6 ++++++ dev/release/verify_rc.sh | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/dev/release/create_rc.sh b/dev/release/create_rc.sh index 54b8596fda..8970aab7b1 100755 --- a/dev/release/create_rc.sh +++ b/dev/release/create_rc.sh @@ -19,6 +19,12 @@ set -Eeuo pipefail +# bash 3.2 (macOS default) does not run the ERR trap for a failing subshell: +# the script still exits on failure but does not report which step failed. +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 +fi + CURRENT_STEP="" on_error() { diff --git a/dev/release/dependencies.sh b/dev/release/dependencies.sh index 5ef8fec628..85e3b29dd2 100755 --- a/dev/release/dependencies.sh +++ b/dev/release/dependencies.sh @@ -19,6 +19,12 @@ set -Eeuo pipefail +# bash 3.2 (macOS default) does not run the ERR trap for a failing subshell: +# the script still exits on failure but does not report which step failed. +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 +fi + # Keep this in sync with CARGO_DENY_VERSION in .github/workflows/dependencies.yml. # The generated DEPENDENCIES.rust.tsv files are version-sensitive, so the local # cargo-deny must match the version CI uses to avoid spurious diffs. diff --git a/dev/release/release.sh b/dev/release/release.sh index 0e54796831..1d1d200666 100755 --- a/dev/release/release.sh +++ b/dev/release/release.sh @@ -19,6 +19,12 @@ set -Eeuo pipefail +# bash 3.2 (macOS default) does not run the ERR trap for a failing subshell: +# the script still exits on failure but does not report which step failed. +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 +fi + CURRENT_STEP="" on_error() { diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index c4a8413f54..2538034523 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -19,6 +19,12 @@ set -Eeuo pipefail +# bash 3.2 (macOS default) does not run the ERR trap for a failing subshell: +# the script still exits on failure but does not report which step failed. +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 +fi + CURRENT_STEP="" VERIFY_SUCCESS="no" VERIFY_OWNS_TMPDIR="0" From aa5ff7e004b01137010923aaac640532d214f94a Mon Sep 17 00:00:00 2001 From: Xander Date: Tue, 15 Sep 2026 10:11:42 +0000 Subject: [PATCH 68/74] fix(encryption): return DataInvalid for invalid AES key sizes (#3226) `AesKeySize::from_key_length` and `AesKeySize::from_str` returned `FeatureUnsupported` for anything outside 128/192/256-bit. AES defines no other key sizes, so these are invalid input rather than something a later version could support, and `FeatureUnsupported` here is indistinguishable from the genuinely unsupported cases such as an unknown key metadata version. `from_key_length` is reachable from `SecureKey::new`, `StandardKeyMetadata::try_new` and the `encryption.data-key-length` table property, all of which now surface `DataInvalid`. Messages change from "Unsupported ..." to "Invalid ..." to match. --- crates/iceberg/src/encryption/crypto.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/encryption/crypto.rs b/crates/iceberg/src/encryption/crypto.rs index 1a4be53544..babdb33bce 100644 --- a/crates/iceberg/src/encryption/crypto.rs +++ b/crates/iceberg/src/encryption/crypto.rs @@ -66,8 +66,8 @@ impl AesKeySize { 24 => Ok(Self::Bits192), 32 => Ok(Self::Bits256), _ => Err(Error::new( - ErrorKind::FeatureUnsupported, - format!("Unsupported data key length: {len} (must be 16, 24, or 32)"), + ErrorKind::DataInvalid, + format!("Invalid data key length: {len} (must be 16, 24, or 32)"), )), } } @@ -82,8 +82,8 @@ impl FromStr for AesKeySize { "192" | "AES_GCM_192" | "AES192_GCM" => Ok(Self::Bits192), "256" | "AES_GCM_256" | "AES256_GCM" => Ok(Self::Bits256), _ => Err(Error::new( - ErrorKind::FeatureUnsupported, - format!("Unsupported AES key size: {s}"), + ErrorKind::DataInvalid, + format!("Invalid AES key size: {s}"), )), } } @@ -295,6 +295,15 @@ mod tests { ); assert!(AesKeySize::from_key_length(8).is_err()); + for len in [0, 8, 15, 20, 33] { + let err = AesKeySize::from_key_length(len).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "for length {len}"); + assert_eq!( + err.message(), + format!("Invalid data key length: {len} (must be 16, 24, or 32)") + ); + } + assert_eq!(AesKeySize::from_str("128").unwrap(), AesKeySize::Bits128); assert_eq!( AesKeySize::from_str("AES_GCM_128").unwrap(), @@ -304,7 +313,11 @@ mod tests { AesKeySize::from_str("AES_GCM_256").unwrap(), AesKeySize::Bits256 ); - assert!(AesKeySize::from_str("INVALID").is_err()); + for size in ["", "127", "AES_GCM_512", "INVALID"] { + let err = AesKeySize::from_str(size).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "for size {size}"); + assert_eq!(err.message(), format!("Invalid AES key size: {size}")); + } } #[test] From 930c10298e43daeb7316b20b83049e9f724ae58c Mon Sep 17 00:00:00 2001 From: Xander Date: Tue, 15 Sep 2026 15:14:51 +0000 Subject: [PATCH 69/74] feat(python) add standard key metadata python bindings (#3206) * add standard ket metadata python bindings * fix * fix: clarify unsupported key metadata version error Report the received and supported key metadata versions, matching the wording expected by PyIceberg. Add Rust and Python regression coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * named struct * improve tests * tests --------- Co-authored-by: Kevin Liu Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/python/src/encryption.rs | 121 ++++++++++++++++++ bindings/python/src/lib.rs | 2 + bindings/python/tests/test_encryption.py | 117 +++++++++++++++++ crates/iceberg/src/encryption/key_metadata.rs | 37 +++++- 4 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 bindings/python/src/encryption.rs create mode 100644 bindings/python/tests/test_encryption.py diff --git a/bindings/python/src/encryption.rs b/bindings/python/src/encryption.rs new file mode 100644 index 0000000000..c0b14875b5 --- /dev/null +++ b/bindings/python/src/encryption.rs @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg::encryption::StandardKeyMetadata; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +use crate::error::to_py_err; + +/// The encryption key, AAD prefix and file length needed to decrypt a single file. +#[pyclass( + frozen, + name = "StandardKeyMetadata", + module = "pyiceberg_core.encryption" +)] +pub struct PyStandardKeyMetadata { + // Python objects, so attribute access increfs rather than copying the key on every read. + encryption_key: Py, + aad_prefix: Option>, + file_length: Option, +} + +#[pymethods] +impl PyStandardKeyMetadata { + #[getter] + fn encryption_key(&self, py: Python<'_>) -> Py { + self.encryption_key.clone_ref(py) + } + + #[getter] + fn aad_prefix(&self, py: Python<'_>) -> Option> { + self.aad_prefix.as_ref().map(|prefix| prefix.clone_ref(py)) + } + + #[getter] + fn file_length(&self) -> Option { + self.file_length + } + + /// Redacts the key, so logging an instance cannot leak key material. + fn __repr__(&self, py: Python<'_>) -> PyResult { + let key_length = self.encryption_key.bind(py).as_bytes().len(); + let aad_prefix = match &self.aad_prefix { + Some(prefix) => prefix.bind(py).repr()?.to_string(), + None => "None".to_string(), + }; + let file_length = match self.file_length { + Some(file_length) => file_length.to_string(), + None => "None".to_string(), + }; + + Ok(format!( + "StandardKeyMetadata(encryption_key=, aad_prefix={aad_prefix}, file_length={file_length})" + )) + } +} + +/// Decode `StandardKeyMetadata` from its wire format. +#[pyfunction] +pub fn decode_standard_key_metadata( + py: Python<'_>, + data: &[u8], +) -> PyResult { + let metadata = StandardKeyMetadata::decode(data).map_err(to_py_err)?; + + Ok(PyStandardKeyMetadata { + encryption_key: PyBytes::new(py, metadata.encryption_key().as_bytes()).unbind(), + aad_prefix: metadata + .aad_prefix() + .map(|aad_prefix| PyBytes::new(py, aad_prefix).unbind()), + file_length: metadata.file_length(), + }) +} + +/// Encode `StandardKeyMetadata` to its wire format. +#[pyfunction] +#[pyo3(signature = (encryption_key, aad_prefix=None, file_length=None))] +pub fn encode_standard_key_metadata<'py>( + py: Python<'py>, + encryption_key: &[u8], + aad_prefix: Option<&[u8]>, + file_length: Option, +) -> PyResult> { + let mut metadata = StandardKeyMetadata::try_new(encryption_key).map_err(to_py_err)?; + + if let Some(aad_prefix) = aad_prefix { + metadata = metadata.with_aad_prefix(aad_prefix); + } + if let Some(file_length) = file_length { + metadata = metadata.with_file_length(file_length); + } + + Ok(PyBytes::new(py, &metadata.encode().map_err(to_py_err)?)) +} + +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let this = PyModule::new(py, "encryption")?; + + this.add_class::()?; + this.add_function(wrap_pyfunction!(decode_standard_key_metadata, &this)?)?; + this.add_function(wrap_pyfunction!(encode_standard_key_metadata, &this)?)?; + + m.add_submodule(&this)?; + py.import("sys")? + .getattr("modules")? + .set_item("pyiceberg_core.encryption", this) +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index baddbe93a0..c50b1ce15a 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -18,6 +18,7 @@ use pyo3::prelude::*; mod data_file; +mod encryption; mod error; mod manifest; mod transform; @@ -26,5 +27,6 @@ mod transform; fn pyiceberg_core_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { transform::register_module(py, m)?; manifest::register_module(py, m)?; + encryption::register_module(py, m)?; Ok(()) } diff --git a/bindings/python/tests/test_encryption.py b/bindings/python/tests/test_encryption.py new file mode 100644 index 0000000000..1e47156077 --- /dev/null +++ b/bindings/python/tests/test_encryption.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from pyiceberg_core import encryption + +AES128_KEY = b"0123456789012345" +# A version byte, then the Avro-encoded encryption_key: zigzag length 16, then the key. +ENCODED_PREFIX = b"\x01\x20" + AES128_KEY + + +def fields(metadata): + return (metadata.encryption_key, metadata.aad_prefix, metadata.file_length) + + +# Pinned in both directions so the encoding stays compatible with the Java and Python +# implementations, including the null union tags written for absent optional fields. +@pytest.mark.parametrize( + "aad_prefix, file_length, encoded", + [ + (None, None, ENCODED_PREFIX + b"\x00\x00"), + (b"ad", None, ENCODED_PREFIX + b"\x02\x04ad\x00"), + (b"ad", 1024, ENCODED_PREFIX + b"\x02\x04ad\x02\x80\x10"), + (b"", None, ENCODED_PREFIX + b"\x02\x00\x00"), + ], +) +def test_wire_format(aad_prefix, file_length, encoded): + assert fields(encryption.decode_standard_key_metadata(encoded)) == ( + AES128_KEY, + aad_prefix, + file_length, + ) + assert ( + encryption.encode_standard_key_metadata(AES128_KEY, aad_prefix, file_length) + == encoded + ) + + +def test_repr_redacts_encryption_key(): + encoded = encryption.encode_standard_key_metadata(AES128_KEY, b"ad", 1024) + + rendered = repr(encryption.decode_standard_key_metadata(encoded)) + + assert AES128_KEY not in rendered.encode() + assert rendered == ( + "StandardKeyMetadata(encryption_key=, " + "aad_prefix=b'ad', file_length=1024)" + ) + + +@pytest.mark.parametrize("key_length", [16, 24, 32]) +def test_encode_accepts_aes_key_lengths(key_length): + encoded = encryption.encode_standard_key_metadata(bytes(key_length)) + metadata = encryption.decode_standard_key_metadata(encoded) + + assert fields(metadata) == (bytes(key_length), None, None) + + +@pytest.mark.parametrize("key_length", [0, 4, 15, 20, 33]) +def test_encode_rejects_invalid_key_length(key_length): + with pytest.raises(ValueError, match="key length"): + encryption.encode_standard_key_metadata(bytes(key_length)) + + +@pytest.mark.parametrize("key_length", [0, 4, 15, 20, 33]) +def test_decode_rejects_invalid_key_length(key_length): + # Key length is validated on decode too, not only on encode. The Avro zigzag length + # is 2 * key_length, a single byte for every length here. + data = b"\x01" + bytes([key_length * 2]) + bytes(key_length) + b"\x00\x00" + + with pytest.raises(ValueError, match="Invalid encryption key in key metadata"): + encryption.decode_standard_key_metadata(data) + + +@pytest.mark.parametrize("data", [b"\x02", b"\x02\x20" + AES128_KEY + b"\x00\x00"]) +def test_decode_rejects_unsupported_version(data): + with pytest.raises(ValueError, match=r"version: 2"): + encryption.decode_standard_key_metadata(data) + + +def test_decode_rejects_empty_buffer(): + with pytest.raises(ValueError, match="Empty key metadata"): + encryption.decode_standard_key_metadata(b"") + + +def test_decode_tolerates_trailing_bytes(): + # Deliberate, matching Java: a reader must tolerate fields a newer writer appended. + encoded = encryption.encode_standard_key_metadata(AES128_KEY, b"ad", 1024) + + metadata = encryption.decode_standard_key_metadata(encoded + b"\xde\xad\xbe\xef") + + assert fields(metadata) == (AES128_KEY, b"ad", 1024) + + +@pytest.mark.xfail( + reason="apache/avro-rs#664: a missing union tag decodes as null; fixed upstream in 0.23", + strict=True, +) +def test_decode_rejects_truncated_union_tags(): + # Version and key, with both union tags omitted, currently decodes as if the + # optional fields were absent rather than erroring. + with pytest.raises(ValueError): + encryption.decode_standard_key_metadata(ENCODED_PREFIX) diff --git a/crates/iceberg/src/encryption/key_metadata.rs b/crates/iceberg/src/encryption/key_metadata.rs index 1169698e2f..765d600b50 100644 --- a/crates/iceberg/src/encryption/key_metadata.rs +++ b/crates/iceberg/src/encryption/key_metadata.rs @@ -206,7 +206,7 @@ mod _serde { if version != V1 { return Err(Error::new( ErrorKind::FeatureUnsupported, - format!("Cannot resolve schema for version: {version}"), + format!("Unsupported key metadata version: {version} (supported: {V1})"), )); } @@ -302,6 +302,10 @@ mod tests { assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert_eq!( + err.message(), + "Unsupported key metadata version: 2 (supported: 1)" + ); } #[test] @@ -358,4 +362,35 @@ mod tests { ); } } + + #[test] + fn test_decode_tolerates_trailing_bytes() { + // Deliberate, not an oversight: Java's RawDecoder reads a single datum without + // checking for EOF, and Iceberg depends on that leniency — `file_length` was + // appended to SCHEMA_V1 without bumping the version byte. Requiring EOF here + // would stop older readers tolerating fields a newer writer appended. + let key = b"0123456789012345"; + let aad = b"1234567890123456"; + let file_length = 1024; + + let serialized = StandardKeyMetadata::try_new(key) + .unwrap() + .with_aad_prefix(aad) + .with_file_length(file_length) + .encode() + .unwrap(); + + // Arbitrary junk, then bytes shaped like a further optional field appended by a + // newer writer (union tag "present", then a 4-byte value). + for trailing in [b"\xde\xad\xbe\xef".as_slice(), b"\x02\x08more".as_slice()] { + let mut extended = serialized.to_vec(); + extended.extend_from_slice(trailing); + + let parsed = StandardKeyMetadata::decode(&extended).unwrap(); + + assert_eq!(parsed.encryption_key().as_bytes(), key); + assert_eq!(parsed.aad_prefix(), Some(aad.as_slice())); + assert_eq!(parsed.file_length(), Some(file_length)); + } + } } From 8193cce4b3cfe4608738cdf36e0ddf74a15a415b Mon Sep 17 00:00:00 2001 From: Xander Date: Tue, 15 Sep 2026 20:17:20 +0000 Subject: [PATCH 70/74] feat: bloom filter pushdown (#2398) * feat: bloom filter pushdown * test * avoid copy * new name * fmt * doc string * handle type widening * widen decimals * pub crate * full import * lifetime * log * add not supported comment * make it more safe * Add tests * fix * fmt * don't read not predicates * tests * remove boolean * doc * update 0 test * UUID * update comment * update test doc --- crates/iceberg/public-api.txt | 2 + crates/iceberg/src/arrow/reader/mod.rs | 19 + crates/iceberg/src/arrow/reader/pipeline.rs | 102 ++ crates/iceberg/src/arrow/reader/row_filter.rs | 479 ++++- .../expr/visitors/bloom_filter_evaluator.rs | 1615 +++++++++++++++++ crates/iceberg/src/expr/visitors/mod.rs | 1 + crates/iceberg/src/scan/mod.rs | 22 +- .../iceberg/src/spec/values/decimal_utils.rs | 100 + 8 files changed, 2338 insertions(+), 2 deletions(-) create mode 100644 crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 9ba84036b7..7d03d6b1fb 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -72,6 +72,7 @@ impl iceberg::arrow::ArrowReaderBuilder pub fn iceberg::arrow::ArrowReaderBuilder::build(self) -> iceberg::arrow::ArrowReader pub fn iceberg::arrow::ArrowReaderBuilder::new(file_io: iceberg::io::FileIO, runtime: iceberg::Runtime) -> Self pub fn iceberg::arrow::ArrowReaderBuilder::with_batch_size(self, batch_size: usize) -> Self +pub fn iceberg::arrow::ArrowReaderBuilder::with_bloom_filter_enabled(self, bloom_filter_enabled: bool) -> Self pub fn iceberg::arrow::ArrowReaderBuilder::with_data_file_concurrency_limit(self, val: usize) -> Self pub fn iceberg::arrow::ArrowReaderBuilder::with_metadata_size_hint(self, metadata_size_hint: usize) -> Self pub fn iceberg::arrow::ArrowReaderBuilder::with_range_coalesce_bytes(self, range_coalesce_bytes: u64) -> Self @@ -1363,6 +1364,7 @@ pub fn iceberg::scan::TableScanBuilder<'a>::select_all(self) -> Self pub fn iceberg::scan::TableScanBuilder<'a>::select_empty(self) -> Self pub fn iceberg::scan::TableScanBuilder<'a>::snapshot_id(self, snapshot_id: i64) -> Self pub fn iceberg::scan::TableScanBuilder<'a>::with_batch_size(self, batch_size: core::option::Option) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::with_bloom_filter_enabled(self, bloom_filter_enabled: bool) -> Self pub fn iceberg::scan::TableScanBuilder<'a>::with_case_sensitive(self, case_sensitive: bool) -> Self pub fn iceberg::scan::TableScanBuilder<'a>::with_concurrency_limit(self, limit: usize) -> Self pub fn iceberg::scan::TableScanBuilder<'a>::with_data_file_concurrency_limit(self, limit: usize) -> Self diff --git a/crates/iceberg/src/arrow/reader/mod.rs b/crates/iceberg/src/arrow/reader/mod.rs index 495283f47f..9d9c8be48e 100644 --- a/crates/iceberg/src/arrow/reader/mod.rs +++ b/crates/iceberg/src/arrow/reader/mod.rs @@ -57,6 +57,7 @@ pub struct ArrowReaderBuilder { concurrency_limit_data_files: usize, row_group_filtering_enabled: bool, row_selection_enabled: bool, + bloom_filter_enabled: bool, parquet_read_options: ParquetReadOptions, runtime: Runtime, } @@ -72,6 +73,7 @@ impl ArrowReaderBuilder { concurrency_limit_data_files: num_cpus, row_group_filtering_enabled: true, row_selection_enabled: false, + bloom_filter_enabled: false, parquet_read_options: ParquetReadOptions::builder().build(), runtime, } @@ -102,6 +104,21 @@ impl ArrowReaderBuilder { self } + /// Determines whether to enable bloom filter-based row group filtering. + /// + /// When enabled, if a read is performed with an equality or IN predicate, + /// the bloom filter for relevant columns in each row group is read and + /// checked. Row groups where the bloom filter proves the value is absent + /// are skipped entirely. + /// + /// Defaults to disabled. Each bloom filter is a separate read, and they are + /// issued serially — one round trip per relevant column per row group, before + /// any data is read. TODO(#3191) + pub fn with_bloom_filter_enabled(mut self, bloom_filter_enabled: bool) -> Self { + self.bloom_filter_enabled = bloom_filter_enabled; + self + } + /// Provide a hint as to the number of bytes to prefetch for parsing the Parquet metadata /// /// This hint can help reduce the number of fetch requests. For more details see the @@ -141,6 +158,7 @@ impl ArrowReaderBuilder { concurrency_limit_data_files: self.concurrency_limit_data_files, row_group_filtering_enabled: self.row_group_filtering_enabled, row_selection_enabled: self.row_selection_enabled, + bloom_filter_enabled: self.bloom_filter_enabled, parquet_read_options: self.parquet_read_options, } } @@ -158,5 +176,6 @@ pub struct ArrowReader { row_group_filtering_enabled: bool, row_selection_enabled: bool, + bloom_filter_enabled: bool, parquet_read_options: ParquetReadOptions, } diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 5354c72043..7ec9bf4d55 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -44,6 +44,10 @@ use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; use crate::arrow::scan_metrics::{CountingFileRead, ScanMetrics, ScanResult}; use crate::encryption::StandardKeyMetadata; use crate::error::Result; +use crate::expr::BoundPredicate; +use crate::expr::visitors::bloom_filter_evaluator::{ + BloomFilterEvaluator, ColumnBloomFilter, collect_bloom_filter_field_ids, +}; use crate::io::{FileIO, FileMetadata, FileRead}; use crate::metadata_columns::{ RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_COL_NAME_POS, @@ -70,6 +74,7 @@ impl ArrowReader { .with_scan_metrics(scan_metrics.clone()), row_group_filtering_enabled: self.row_group_filtering_enabled, row_selection_enabled: self.row_selection_enabled, + bloom_filter_enabled: self.bloom_filter_enabled, parquet_read_options: self.parquet_read_options, scan_metrics: scan_metrics.clone(), }; @@ -123,6 +128,7 @@ struct FileScanTaskReader { delete_file_loader: CachingDeleteFileLoader, row_group_filtering_enabled: bool, row_selection_enabled: bool, + bloom_filter_enabled: bool, parquet_read_options: ParquetReadOptions, scan_metrics: ScanMetrics, } @@ -632,6 +638,30 @@ impl FileScanTaskReader { }; } + if self.bloom_filter_enabled { + let all_rgs; + let candidate_rgs = match &selected_row_group_indices { + Some(indices) => indices.as_slice(), + None => { + all_rgs = (0..record_batch_stream_builder.metadata().num_row_groups()) + .collect::>(); + &all_rgs + } + }; + + let bloom_filtered = Self::filter_row_groups_by_bloom_filter( + &predicate, + &mut record_batch_stream_builder, + candidate_rgs, + &field_id_map, + ) + .await?; + + if bloom_filtered.len() < candidate_rgs.len() { + selected_row_group_indices = Some(bloom_filtered); + } + } + if self.row_selection_enabled { row_selection = ArrowReader::get_row_selection_for_filter_predicate( &predicate, @@ -692,6 +722,78 @@ impl FileScanTaskReader { Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream) } + + /// Reads bloom filters for relevant columns and evaluates the predicate + /// against them to filter out row groups that definitely don't match. + async fn filter_row_groups_by_bloom_filter( + predicate: &BoundPredicate, + builder: &mut ParquetRecordBatchStreamBuilder, + candidate_row_groups: &[usize], + field_id_map: &HashMap, + ) -> Result> { + // Only collect field IDs from eq/in predicates — the only types + // bloom filters can help with. Skip columns not in the parquet schema. + let bloom_filter_field_ids: Vec = collect_bloom_filter_field_ids(predicate)? + .into_iter() + .filter(|id| field_id_map.contains_key(id)) + .collect(); + + if bloom_filter_field_ids.is_empty() { + return Ok(candidate_row_groups.to_vec()); + } + + let mut result = Vec::with_capacity(candidate_row_groups.len()); + + for &rg_idx in candidate_row_groups { + let mut bloom_filters: HashMap = HashMap::new(); + + for &field_id in &bloom_filter_field_ids { + let col_idx = field_id_map[&field_id]; + let col_meta = builder.metadata().row_group(rg_idx).column(col_idx); + + // Only attempt to load if this column chunk actually has a bloom filter + if col_meta.bloom_filter_offset().is_none() { + continue; + } + + let physical_type = col_meta.column_type(); + let type_length = col_meta.column_descr().type_length(); + + match builder + .get_row_group_column_bloom_filter(rg_idx, col_idx) + .await + { + Ok(Some(sbbf)) => { + bloom_filters.insert( + field_id, + ColumnBloomFilter::new(sbbf, physical_type, type_length), + ); + } + Ok(None) => {} + Err(e) => { + // Left absent from the map, so the evaluator treats the column + // as might-match and the row group survives. + tracing::debug!( + "Bloom filter for field {field_id} in row group {rg_idx} could not be read: {e}" + ); + } + } + } + + match BloomFilterEvaluator::eval(predicate, &bloom_filters) { + Ok(true) => result.push(rg_idx), + Ok(false) => { /* Row group pruned by bloom filter */ } + Err(e) => { + tracing::debug!( + "Bloom filter evaluation failed for row group {rg_idx}, including it: {e}" + ); + result.push(rg_idx); + } + } + } + + Ok(result) + } } impl ArrowReader { diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 389231c461..34b7391134 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -217,7 +217,8 @@ mod tests { use arrow_array::cast::AsArray; use arrow_array::{ - ArrayRef, Int32Array, Int64Array, LargeStringArray, RecordBatch, StringArray, + ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, + RecordBatch, StringArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; @@ -1280,4 +1281,480 @@ mod tests { "positional deletes must be applied correctly even when page indexes are absent" ); } + + // Bloom filter pushdown: on-vs-off equivalence + // Pushdown must never change results. An encoding bug in the probe shows up as + // rows the bloom filter drops and the row filter keeps, so every case below + // reads the same file twice and compares. Fixtures spread each row group's + // values across the whole domain, so min/max statistics cannot prune and any + // reduction in bytes read is attributable to the bloom filter. + + const ROWS_PER_GROUP: i32 = 20_000; + const GROUPS: i32 = 3; + /// Gap between consecutive values in a row group, so a value can be absent from + /// the file yet still sit inside every row group's min/max range. + const STRIDE: i32 = 30; + + /// The `i`th value of row group `group`, spread across the whole domain. + fn interleaved(group: i32, i: i32) -> i32 { + i * STRIDE + group + } + + fn field_with_id(name: &str, ty: DataType, id: i32) -> Field { + Field::new(name, ty, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + id.to_string(), + )])) + } + + /// Writes one row group per batch, optionally with bloom filters. + fn write_row_groups( + path: &str, + arrow_schema: Arc, + row_groups: Vec, + with_bloom_filters: bool, + ) { + let mut props = WriterProperties::builder().set_compression(Compression::UNCOMPRESSED); + if with_bloom_filters { + props = props + .set_bloom_filter_enabled(true) + .set_bloom_filter_max_ndv(ROWS_PER_GROUP as u64); + } + + let file = File::create(path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props.build())).unwrap(); + for batch in row_groups { + writer.write(&batch).unwrap(); + // Force a row group boundary so each batch is independently prunable. + writer.flush().unwrap(); + } + writer.close().unwrap(); + } + + async fn read_once( + file_path: &str, + schema: Arc, + project_field_ids: Vec, + predicate: Predicate, + bloom_enabled: bool, + ) -> (Vec, u64) { + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()) + .with_bloom_filter_enabled(bloom_enabled) + // Keep the fixed footer prefetch small so the byte measurement reflects + // row group I/O rather than a constant metadata read. + .with_metadata_size_hint(8 * 1024) + .build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids) + .with_predicate(Some(predicate.bind(schema, true).unwrap())) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader.read(tasks).unwrap(); + let metrics = result.metrics().clone(); + let batches: Vec = result.stream().try_collect().await.unwrap(); + + (batches, metrics.bytes_read()) + } + + /// Batch boundaries differ when fewer row groups are read, so collapse to a + /// single batch before comparing. `None` means no rows at all. + fn collapse(batches: &[RecordBatch]) -> Option { + let first = batches.first()?; + Some(arrow_select::concat::concat_batches(&first.schema(), batches).unwrap()) + } + + /// Reads the file with pushdown on and off, asserts the rows are identical, and + /// returns the rows plus (bytes_on, bytes_off). + async fn assert_pushdown_agrees( + case: &str, + file_path: &str, + schema: Arc, + project_field_ids: Vec, + predicate: Predicate, + ) -> (Option, u64, u64) { + let (off, bytes_off) = read_once( + file_path, + schema.clone(), + project_field_ids.clone(), + predicate.clone(), + false, + ) + .await; + let (on, bytes_on) = read_once(file_path, schema, project_field_ids, predicate, true).await; + + let off = collapse(&off); + let on = collapse(&on); + assert_eq!( + on, off, + "{case}: bloom filter pushdown changed the rows returned" + ); + + (on, bytes_on, bytes_off) + } + + fn rows(batch: &Option) -> usize { + batch.as_ref().map_or(0, |b| b.num_rows()) + } + + fn value_schema(iceberg_type: Type) -> Arc { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![NestedField::required(1, "v", iceberg_type).into()]) + .build() + .unwrap(), + ) + } + + /// Writes a single-column file, one row group per group index. + fn write_value_fixture( + path: &str, + data_type: DataType, + make_group: impl Fn(i32) -> ArrayRef, + with_bloom_filters: bool, + ) { + let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id("v", data_type, 1)])); + let row_groups = (0..GROUPS) + .map(|g| RecordBatch::try_new(arrow_schema.clone(), vec![make_group(g)]).unwrap()) + .collect(); + write_row_groups(path, arrow_schema, row_groups, with_bloom_filters); + } + + /// The core check for one physical encoding: a value that is present must + /// survive pushdown, and one that is absent must prune every row group. The + /// first catches a probe that encodes wrongly and drops real rows; the second + /// catches a probe that silently never prunes. + async fn assert_prunes_and_agrees( + case: &str, + file_path: &str, + schema: Arc, + present: Datum, + absent: Datum, + ) { + let (on, bytes_on, bytes_off) = assert_pushdown_agrees( + &format!("{case}/present"), + file_path, + schema.clone(), + vec![1], + Reference::new("v").equal_to(present), + ) + .await; + assert_eq!(rows(&on), 1, "{case}/present: expected one matching row"); + assert!( + bytes_on < bytes_off, + "{case}/present: pushdown must skip row groups: {bytes_on} vs {bytes_off}" + ); + + let (on, bytes_on, bytes_off) = assert_pushdown_agrees( + &format!("{case}/absent"), + file_path, + schema, + vec![1], + Reference::new("v").equal_to(absent), + ) + .await; + assert_eq!(rows(&on), 0, "{case}/absent: expected no matching rows"); + assert!( + bytes_on * 2 < bytes_off, + "{case}/absent: pruning every row group should cut reads sharply: \ + {bytes_on} vs {bytes_off}" + ); + } + + /// Value present in exactly one row group: `interleaved(1, 51)`. + const PRESENT_INT: i32 = 51 * STRIDE + 1; + /// Not congruent to any group index mod STRIDE, so absent from the file while + /// still inside every row group's min/max range. + const ABSENT_INT: i32 = 7; + + fn int_group(g: i32) -> ArrayRef { + Arc::new(Int32Array::from( + (0..ROWS_PER_GROUP) + .map(|i| interleaved(g, i)) + .collect::>(), + )) + } + + #[tokio::test] + async fn test_bloom_pushdown_int32_eq() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/int32.parquet", tmp.path().to_str().unwrap()); + write_value_fixture(&path, DataType::Int32, int_group, true); + + assert_prunes_and_agrees( + "int32", + &path, + value_schema(Type::Primitive(PrimitiveType::Int)), + Datum::int(PRESENT_INT), + Datum::int(ABSENT_INT), + ) + .await; + } + + /// `IN` takes a different evaluator path from `eq`: it must keep the row group + /// when any literal might be present, and prune only when all are absent. + #[tokio::test] + async fn test_bloom_pushdown_int32_in() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/int32_in.parquet", tmp.path().to_str().unwrap()); + write_value_fixture(&path, DataType::Int32, int_group, true); + let schema = value_schema(Type::Primitive(PrimitiveType::Int)); + + // One present literal keeps its row group; the absent one must not suppress it. + let (on, bytes_on, bytes_off) = assert_pushdown_agrees( + "int32_in/mixed", + &path, + schema.clone(), + vec![1], + Reference::new("v").is_in([Datum::int(PRESENT_INT), Datum::int(ABSENT_INT)]), + ) + .await; + assert_eq!(rows(&on), 1); + assert!(bytes_on < bytes_off, "{bytes_on} vs {bytes_off}"); + + // All literals absent: every row group prunes. + let (on, bytes_on, bytes_off) = assert_pushdown_agrees( + "int32_in/all_absent", + &path, + schema, + vec![1], + Reference::new("v").is_in([Datum::int(ABSENT_INT), Datum::int(ABSENT_INT + 1)]), + ) + .await; + assert_eq!(rows(&on), 0); + assert!(bytes_on * 2 < bytes_off, "{bytes_on} vs {bytes_off}"); + } + + #[tokio::test] + async fn test_bloom_pushdown_string_eq() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/string.parquet", tmp.path().to_str().unwrap()); + write_value_fixture( + &path, + DataType::Utf8, + |g| { + Arc::new(StringArray::from( + (0..ROWS_PER_GROUP) + .map(|i| format!("{:016}", interleaved(g, i))) + .collect::>(), + )) + }, + true, + ); + + assert_prunes_and_agrees( + "string", + &path, + value_schema(Type::Primitive(PrimitiveType::String)), + Datum::string(format!("{PRESENT_INT:016}")), + Datum::string(format!("{ABSENT_INT:016}")), + ) + .await; + } + + /// Decimal precision decides the Parquet physical type, and the probe has to be + /// encoded to match: precision 9 -> INT32, 15 -> INT64, 25 -> FIXED_LEN_BYTE_ARRAY. + /// Each width is a separate arm of `check_in_bloom_filter`. + async fn check_decimal_width(name: &str, precision: u32) { + const SCALE: u32 = 2; + let tmp = TempDir::new().unwrap(); + let path = format!("{}/{name}.parquet", tmp.path().to_str().unwrap()); + + write_value_fixture( + &path, + DataType::Decimal128(precision as u8, SCALE as i8), + |g| { + let values: Vec = (0..ROWS_PER_GROUP) + .map(|i| i128::from(interleaved(g, i))) + .collect(); + Arc::new( + Decimal128Array::from(values) + .with_precision_and_scale(precision as u8, SCALE as i8) + .unwrap(), + ) + }, + true, + ); + + let datum = |mantissa: i32| { + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale( + i128::from(mantissa), + SCALE, + ), + precision, + ) + .unwrap() + }; + + assert_prunes_and_agrees( + name, + &path, + value_schema(Type::Primitive(PrimitiveType::Decimal { + precision, + scale: SCALE, + })), + datum(PRESENT_INT), + datum(ABSENT_INT), + ) + .await; + } + + #[tokio::test] + async fn test_bloom_pushdown_decimal_int32() { + check_decimal_width("decimal_int32", 9).await; + } + + #[tokio::test] + async fn test_bloom_pushdown_decimal_int64() { + check_decimal_width("decimal_int64", 15).await; + } + + #[tokio::test] + async fn test_bloom_pushdown_decimal_fixed_len_byte_array() { + check_decimal_width("decimal_flba", 25).await; + } + + /// Widening a decimal's precision does not rewrite existing files, so the file + /// keeps its original physical width while the bound literal arrives at the new + /// precision. Deriving the probe encoding from the schema instead of the file + /// would miss every entry the writer inserted and prune matching row groups. + #[tokio::test] + async fn test_bloom_pushdown_decimal_widened_precision() { + const SCALE: u32 = 2; + let tmp = TempDir::new().unwrap(); + let path = format!("{}/decimal_widened.parquet", tmp.path().to_str().unwrap()); + + // File written at precision 9 -> INT32. + write_value_fixture( + &path, + DataType::Decimal128(9, SCALE as i8), + |g| { + let values: Vec = (0..ROWS_PER_GROUP) + .map(|i| i128::from(interleaved(g, i))) + .collect(); + Arc::new( + Decimal128Array::from(values) + .with_precision_and_scale(9, SCALE as i8) + .unwrap(), + ) + }, + true, + ); + + // Table schema has since widened to precision 30. + let schema = value_schema(Type::Primitive(PrimitiveType::Decimal { + precision: 30, + scale: SCALE, + })); + let datum = |mantissa: i32| { + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale( + i128::from(mantissa), + SCALE, + ), + 30, + ) + .unwrap() + }; + + assert_prunes_and_agrees( + "decimal_widened", + &path, + schema, + datum(PRESENT_INT), + datum(ABSENT_INT), + ) + .await; + } + + /// `Sbbf` hashes raw IEEE bytes, so `-0.0` and `0.0` are distinct entries. Arrow's + /// float equality is bitwise too, for an independent reason: `ArrowNativeTypeOp::is_eq` + /// implements total-order (`total_cmp`) semantics as `to_bits() == to_bits()`, so + /// `-0.0 != 0.0` and `NaN == NaN` there. That agreement is what makes it safe to prune + /// a row group holding only the opposite zero: if `is_eq` ever became IEEE-lenient, + /// those pruned rows would be rows the row filter should have returned. A probe that + /// canonicalized the sign instead only over-keeps, costing pruning rather than + /// correctness — but it would also hide such a change from this test. + /// + /// Each row group pairs its zero with a larger filler value so its min/max range spans + /// the other zero. Neither statistics path can prune it, so the bloom-off read really + /// does evaluate `is_eq` against `ROWS_PER_GROUP - 1` copies of the opposite zero: that + /// arm is the control pinning arrow's semantics, and IEEE-lenient equality would double + /// the row count it returns. Pairing with a filler also avoids relying on the writer + /// widening a single-value group's bounds to `[-0.0, +0.0]`. + #[tokio::test] + async fn test_bloom_pushdown_float_signed_zero() { + const FILLER: f32 = 5.0; + let tmp = TempDir::new().unwrap(); + let path = format!("{}/float_zero.parquet", tmp.path().to_str().unwrap()); + + // Row group 0 holds -0.0 and never +0.0; row group 1 the reverse; row group 2 + // neither zero. + write_value_fixture( + &path, + DataType::Float32, + |g| { + let zero = match g { + 0 => -0.0f32, + 1 => 0.0f32, + _ => 7.5f32, + }; + let mut values = vec![zero; ROWS_PER_GROUP as usize - 1]; + values.push(FILLER); + Arc::new(Float32Array::from(values)) + }, + true, + ); + + let schema = value_schema(Type::Primitive(PrimitiveType::Float)); + let zero_rows = ROWS_PER_GROUP as usize - 1; + + for (label, probe) in [("positive_zero", 0.0f32), ("negative_zero", -0.0f32)] { + let (on, _, _) = assert_pushdown_agrees( + label, + &path, + schema.clone(), + vec![1], + Reference::new("v").equal_to(Datum::float(probe)), + ) + .await; + // Only the group holding this exact zero matches, on both paths. + assert_eq!( + rows(&on), + zero_rows, + "{label}: expected only the row group holding this zero to match" + ); + } + } + + /// A file with no bloom filters at all must read identically with the option on, + /// exercising the conservative might-match path end to end. + #[tokio::test] + async fn test_bloom_pushdown_file_without_bloom_filters() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/no_filters.parquet", tmp.path().to_str().unwrap()); + write_value_fixture(&path, DataType::Int32, int_group, false); + + let (on, _, _) = assert_pushdown_agrees( + "no_filters", + &path, + value_schema(Type::Primitive(PrimitiveType::Int)), + vec![1], + Reference::new("v").equal_to(Datum::int(PRESENT_INT)), + ) + .await; + assert_eq!(rows(&on), 1); + } } diff --git a/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs new file mode 100644 index 0000000000..3dfc8ec4a9 --- /dev/null +++ b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs @@ -0,0 +1,1615 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Evaluates predicates against Parquet bloom filters to determine whether +//! a row group can be skipped. + +use std::collections::{HashMap, HashSet}; + +use fnv::FnvHashSet; +use parquet::basic::Type as PhysicalType; +use parquet::bloom_filter::Sbbf; +use parquet::data_type::ByteArray; + +use crate::Result; +use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; +use crate::expr::{BoundPredicate, BoundReference}; +use crate::spec::decimal_utils::decimal_to_fixed_length_bytes_exact; +use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType, Type}; + +const ROW_GROUP_MIGHT_MATCH: Result = Ok(true); +const ROW_GROUP_CANT_MATCH: Result = Ok(false); + +/// A column's bloom filter for one row group, together with the file's physical +/// encoding of that column. A probe must be encoded the way the writer encoded +/// the values it inserted, so the encoding travels with the filter. +pub(crate) struct ColumnBloomFilter { + sbbf: Sbbf, + physical_type: PhysicalType, + /// `type_length` from the file's column descriptor. Only meaningful for + /// `FIXED_LEN_BYTE_ARRAY`. + type_length: i32, +} + +impl ColumnBloomFilter { + pub(crate) fn new(sbbf: Sbbf, physical_type: PhysicalType, type_length: i32) -> Self { + Self { + sbbf, + physical_type, + type_length, + } + } +} + +pub(crate) struct BloomFilterEvaluator<'a> { + /// Maps Iceberg field_id -> bloom filter for this row group + bloom_filters: &'a HashMap, +} + +impl BloomFilterEvaluator<'_> { + /// Evaluate the predicate against the provided bloom filters. + /// Returns `false` if the row group definitely does not match, + /// `true` if it might match. + pub(crate) fn eval( + filter: &BoundPredicate, + bloom_filters: &HashMap, + ) -> Result { + if bloom_filters.is_empty() { + return ROW_GROUP_MIGHT_MATCH; + } + + let mut evaluator = BloomFilterEvaluator { bloom_filters }; + visit(&mut evaluator, filter) + } + + fn check_datum(&self, reference: &BoundReference, datum: &Datum) -> bool { + let field_id = reference.field().id; + let Some(column) = self.bloom_filters.get(&field_id) else { + // No bloom filter for this column — conservatively might match + return true; + }; + + check_in_bloom_filter(column, datum) + } +} + +/// Collects field IDs that appear in `eq` or `in` predicates — the only +/// predicate types that benefit from bloom filter checks. +/// +/// Each node returns the field IDs its subtree contributes, mirroring +/// [`BloomFilterEvaluator`]'s structure so that a field is collected only where +/// the evaluator can act on it. In particular `not` discards its subtree: the +/// evaluator's `not` returns might-match regardless, so a filter fetched for a +/// field under a `NOT` could never prune and would be pure wasted I/O. +/// +/// Boolean fields are skipped for the same reason: with only two possible values, +/// a column chunk's min/max statistics already determine membership exactly, so a +/// bloom filter cannot prune anything statistics did not, and reading one costs a +/// round trip to learn nothing. +pub(crate) fn collect_bloom_filter_field_ids(predicate: &BoundPredicate) -> Result> { + visit(&mut BloomFilterFieldIdCollector, predicate) +} + +struct BloomFilterFieldIdCollector; + +/// Field IDs worth fetching a bloom filter for: none if probing the column could +/// never prune. +fn probeable_field_id(reference: &BoundReference) -> HashSet { + if matches!( + reference.field().field_type.as_ref(), + Type::Primitive(PrimitiveType::Boolean) + ) { + return HashSet::new(); + } + HashSet::from([reference.field().id]) +} + +impl BoundPredicateVisitor for BloomFilterFieldIdCollector { + type T = HashSet; + + fn always_true(&mut self) -> Result { + Ok(HashSet::new()) + } + + fn always_false(&mut self) -> Result { + Ok(HashSet::new()) + } + + fn and(&mut self, mut lhs: Self::T, rhs: Self::T) -> Result { + lhs.extend(rhs); + Ok(lhs) + } + + fn or(&mut self, mut lhs: Self::T, rhs: Self::T) -> Result { + lhs.extend(rhs); + Ok(lhs) + } + + fn not(&mut self, _inner: Self::T) -> Result { + Ok(HashSet::new()) + } + + fn is_null(&mut self, _r: &BoundReference, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn not_null(&mut self, _r: &BoundReference, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn is_nan(&mut self, _r: &BoundReference, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn not_nan(&mut self, _r: &BoundReference, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn less_than( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn less_than_or_eq( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn greater_than( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn greater_than_or_eq( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn eq(&mut self, r: &BoundReference, _l: &Datum, _p: &BoundPredicate) -> Result { + Ok(probeable_field_id(r)) + } + + fn not_eq(&mut self, _r: &BoundReference, _l: &Datum, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn starts_with( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn not_starts_with( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn r#in( + &mut self, + r: &BoundReference, + _literals: &FnvHashSet, + _p: &BoundPredicate, + ) -> Result { + Ok(probeable_field_id(r)) + } + + fn not_in( + &mut self, + _r: &BoundReference, + _literals: &FnvHashSet, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } +} + +/// Check whether a datum value might be present in the bloom filter. +/// +/// The value must be checked using the same physical encoding the Parquet +/// writer used when inserting into the bloom filter. We use the actual +/// physical type from the column metadata to ensure correctness regardless +/// of which writer produced the file. +fn check_in_bloom_filter(column: &ColumnBloomFilter, datum: &Datum) -> bool { + let ColumnBloomFilter { + sbbf, + physical_type, + type_length, + } = column; + let physical_type = *physical_type; + + match datum.literal() { + // Parquet does not define bloom filter semantics for BOOLEAN — parquet-java + // only hashes int/long/float/double/binary. arrow-rs does write them, but min/max + // statistics already decide membership for a two-valued domain, so there is nothing to gain. + PrimitiveLiteral::Boolean(_) => true, + // A promoted column (int -> long, float -> double) keeps its original + // physical width in files written before the promotion, and the writer + // hashed that width, so probe at the file's width rather than the + // predicate's. + PrimitiveLiteral::Int(v) => match physical_type { + PhysicalType::INT32 => sbbf.check(v), + PhysicalType::INT64 => sbbf.check(&i64::from(*v)), + _ => true, + }, + PrimitiveLiteral::Long(v) => match physical_type { + PhysicalType::INT64 => sbbf.check(v), + PhysicalType::INT32 => match i32::try_from(*v) { + Ok(narrowed) => sbbf.check(&narrowed), + // Too wide for an INT32 column to hold, so the value is provably + // absent and pruning here would be sound; kept conservative for now. + Err(_) => true, + }, + _ => true, + }, + PrimitiveLiteral::Float(v) => match physical_type { + PhysicalType::FLOAT => sbbf.check(&v.0), + PhysicalType::DOUBLE => sbbf.check(&f64::from(v.0)), + _ => true, + }, + PrimitiveLiteral::Double(v) => match physical_type { + PhysicalType::DOUBLE => sbbf.check(&v.0), + PhysicalType::FLOAT => { + let narrowed = v.0 as f32; + // Only an exactly representable value can equal a widened f32. + if f64::from(narrowed) == v.0 { + sbbf.check(&narrowed) + } else { + true + } + } + _ => true, + }, + PrimitiveLiteral::String(v) => sbbf.check(v.as_str()), + PrimitiveLiteral::Binary(v) => sbbf.check(v.as_slice()), + PrimitiveLiteral::Int128(v) => { + // Decimal: dispatch based on the actual Parquet physical type + // from the file, not inferred from precision. + match physical_type { + // Narrow only when the mantissa round-trips; a truncated copy would + // hash to an unrelated slot. + PhysicalType::INT32 => match i32::try_from(*v) { + Ok(narrowed) => sbbf.check(&narrowed), + Err(_) => true, + }, + PhysicalType::INT64 => match i64::try_from(*v) { + Ok(narrowed) => sbbf.check(&narrowed), + Err(_) => true, + }, + PhysicalType::FIXED_LEN_BYTE_ARRAY => { + // Encode to the file's declared length, not one derived from + // the Iceberg precision: a widened precision would change the + // length and miss every entry the writer inserted. + match usize::try_from(*type_length) + .ok() + .and_then(|len| decimal_to_fixed_length_bytes_exact(*v, len)) + { + Some(bytes) => sbbf.check(&ByteArray::from(bytes)), + // Unusable length, or a value too large for the column to + // hold — conservatively might match. + None => true, + } + } + // Known gap: BYTE_ARRAY decimals are valid in Parquet (though not in + // Iceberg's Appendix A) and some older Spark writers emit them. Not + // probed because BYTE_ARRAY carries no `type_length` and sign-extension + // padding is writer-dependent, so a single-length probe would miss + // padded entries and prune row groups that do hold the value. A sound + // version ORs a check over every length from minimal..=16. + _ => true, // Conservatively might match + } + } + PrimitiveLiteral::UInt128(v) => { + // UUID: stored as FIXED_LEN_BYTE_ARRAY(16), big-endian + let bytes = v.to_be_bytes(); + sbbf.check(bytes.as_slice()) + } + PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => true, + } +} + +impl BoundPredicateVisitor for BloomFilterEvaluator<'_> { + type T = bool; + + fn always_true(&mut self) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn always_false(&mut self) -> Result { + ROW_GROUP_CANT_MATCH + } + + fn and(&mut self, lhs: Self::T, rhs: Self::T) -> Result { + Ok(lhs && rhs) + } + + fn or(&mut self, lhs: Self::T, rhs: Self::T) -> Result { + Ok(lhs || rhs) + } + + fn not(&mut self, _inner: Self::T) -> Result { + // Bloom filters are not invertible — we cannot prove presence, + // so NOT of any result must conservatively return "might match". + ROW_GROUP_MIGHT_MATCH + } + + fn is_null( + &mut self, + _reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn not_null( + &mut self, + _reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn is_nan( + &mut self, + _reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn not_nan( + &mut self, + _reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn less_than( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn less_than_or_eq( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn greater_than( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn greater_than_or_eq( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn eq( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + if self.check_datum(reference, literal) { + ROW_GROUP_MIGHT_MATCH + } else { + ROW_GROUP_CANT_MATCH + } + } + + fn not_eq( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn starts_with( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn not_starts_with( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn r#in( + &mut self, + reference: &BoundReference, + literals: &FnvHashSet, + _predicate: &BoundPredicate, + ) -> Result { + let field_id = reference.field().id; + let Some(column) = self.bloom_filters.get(&field_id) else { + return ROW_GROUP_MIGHT_MATCH; + }; + + // If ANY literal might be present, the row group might match + for literal in literals { + if check_in_bloom_filter(column, literal) { + return ROW_GROUP_MIGHT_MATCH; + } + } + + // All literals are definitely absent + ROW_GROUP_CANT_MATCH + } + + fn not_in( + &mut self, + _reference: &BoundReference, + _literals: &FnvHashSet, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::ops::Not; + + use parquet::basic::Type as PhysicalType; + use parquet::bloom_filter::Sbbf; + use parquet::data_type::ByteArray; + + use super::{BloomFilterEvaluator, ColumnBloomFilter, collect_bloom_filter_field_ids}; + use crate::expr::{Bind, BoundPredicate, Reference}; + use crate::spec::decimal_utils::decimal_to_fixed_length_bytes_exact; + use crate::spec::{Datum, NestedField, PrimitiveType, Schema, Type}; + + fn create_test_schema() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap() + } + + fn create_bloom_filter_with_values_i32(values: &[i32]) -> Sbbf { + let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64, 0.01).unwrap(); + for v in values { + sbbf.insert(v); + } + sbbf + } + + fn create_bloom_filter_with_values_str(values: &[&str]) -> Sbbf { + let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64, 0.01).unwrap(); + for v in values { + sbbf.insert(*v); + } + sbbf + } + + #[test] + fn test_eq_value_present() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::int(2)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Row group should might-match when value is present"); + } + + #[test] + fn test_eq_value_absent() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::int(999)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "Row group should not match when value is absent from bloom filter" + ); + } + + #[test] + fn test_eq_no_bloom_filter_for_column() { + let schema = create_test_schema(); + let bloom_filters = HashMap::new(); // No bloom filters + + let predicate = Reference::new("id") + .equal_to(Datum::int(1)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Row group should might-match when no bloom filter available" + ); + } + + #[test] + fn test_in_all_absent() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .is_in([Datum::int(100), Datum::int(200), Datum::int(300)]) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "Row group should not match when all IN values are absent" + ); + } + + #[test] + fn test_in_some_present() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .is_in([Datum::int(2), Datum::int(200)]) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Row group should might-match when at least one IN value is present" + ); + } + + #[test] + fn test_and_one_absent() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([ + ( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + ), + ( + 2, + ColumnBloomFilter::new( + create_bloom_filter_with_values_str(&["alice", "bob"]), + PhysicalType::BYTE_ARRAY, + 0, + ), + ), + ]); + + // id = 999 AND name = 'alice' + // id=999 is absent, so AND should be false + let predicate = Reference::new("id") + .equal_to(Datum::int(999)) + .and(Reference::new("name").equal_to(Datum::string("alice"))) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "AND should be false when one operand is definitely absent" + ); + } + + #[test] + fn test_or_one_present() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + // id = 999 OR id = 2 + // id=2 is present, so OR should be true + let predicate = Reference::new("id") + .equal_to(Datum::int(999)) + .or(Reference::new("id").equal_to(Datum::int(2))) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "OR should be true when one operand might match"); + } + + #[test] + fn test_not_always_might_match() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + // NOT(id = 999) — even though 999 is absent, NOT should still return true + let predicate = Reference::new("id") + .equal_to(Datum::int(999)) + .not() + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "NOT should always return might-match"); + } + + // --- Field ID collection --- + // + // The collector decides which bloom filters get fetched, one round trip per + // column per row group, so anything it reports that the evaluator cannot act + // on is wasted I/O. + + fn collected_ids(predicate: BoundPredicate) -> Vec { + let mut ids: Vec = collect_bloom_filter_field_ids(&predicate) + .unwrap() + .into_iter() + .collect(); + ids.sort_unstable(); + ids + } + + #[test] + fn test_collects_eq_and_in_field_ids() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .equal_to(Datum::int(1)) + .and(Reference::new("name").is_in([Datum::string("alice")])) + .bind(schema.into(), true) + .unwrap(); + + assert_eq!(collected_ids(predicate), vec![1, 2]); + } + + /// `not` discards its subtree, so `eq`/`in` beneath a `NOT` are not collected. + /// The evaluator's `not` returns might-match regardless, so fetching those + /// filters could never prune. + #[test] + fn test_does_not_collect_eq_under_not() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .equal_to(Datum::int(1)) + .not() + .bind(schema.into(), true) + .unwrap(); + + assert!(collected_ids(predicate).is_empty()); + } + + #[test] + fn test_does_not_collect_in_under_not() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .is_in([Datum::int(1), Datum::int(2)]) + .not() + .bind(schema.into(), true) + .unwrap(); + + assert!(collected_ids(predicate).is_empty()); + } + + /// A `NOT` must not suppress collection for its siblings. + #[test] + fn test_collects_sibling_of_not() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .equal_to(Datum::int(1)) + .not() + .and(Reference::new("name").equal_to(Datum::string("alice"))) + .bind(schema.into(), true) + .unwrap(); + + assert_eq!(collected_ids(predicate), vec![2]); + } + + /// A boolean column's min/max statistics already determine membership exactly, + /// so fetching a bloom filter for one can never prune and is pure wasted I/O. + #[test] + fn test_does_not_collect_boolean_field_ids() { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "flag", Type::Primitive(PrimitiveType::Boolean)).into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let predicate = Reference::new("flag") + .equal_to(Datum::bool(true)) + .and(Reference::new("flag").is_in([Datum::bool(false)])) + .and(Reference::new("id").equal_to(Datum::int(1))) + .bind(schema.into(), true) + .unwrap(); + + assert_eq!( + collected_ids(predicate), + vec![2], + "only the non-boolean field should be collected" + ); + } + + /// Even given a filter, a boolean probe stays conservative: Parquet does not + /// define bloom filter semantics for BOOLEAN, so the encoding is not portable. + #[test] + fn test_boolean_probe_never_prunes() { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "flag", Type::Primitive(PrimitiveType::Boolean)).into(), + ]) + .build() + .unwrap(); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&true); + let filters = HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::BOOLEAN, 0))]); + + // `false` was never inserted, yet the row group must survive. + let predicate = Reference::new("flag") + .equal_to(Datum::bool(false)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!(result, "boolean probes must never prune"); + } + + /// Range and `not_eq` predicates cannot be probed, so they contribute nothing. + #[test] + fn test_does_not_collect_unprobeable_operators() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .less_than(Datum::int(1)) + .and(Reference::new("id").not_equal_to(Datum::int(2))) + .and(Reference::new("name").is_not_null()) + .bind(schema.into(), true) + .unwrap(); + + assert!(collected_ids(predicate).is_empty()); + } + + #[test] + fn test_range_predicates_always_might_match() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .less_than(Datum::int(0)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Range predicates should always return might-match"); + } + + #[test] + fn test_string_eq_present() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 2, + ColumnBloomFilter::new( + create_bloom_filter_with_values_str(&["alice", "bob"]), + PhysicalType::BYTE_ARRAY, + 0, + ), + )]); + + let predicate = Reference::new("name") + .equal_to(Datum::string("alice")) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Should might-match when string is in bloom filter"); + } + + #[test] + fn test_string_eq_absent() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 2, + ColumnBloomFilter::new( + create_bloom_filter_with_values_str(&["alice", "bob"]), + PhysicalType::BYTE_ARRAY, + 0, + ), + )]); + + let predicate = Reference::new("name") + .equal_to(Datum::string("charlie")) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "Should not match when string is absent from bloom filter" + ); + } + + // --- Decimal tests --- + + fn create_decimal_schema(precision: u32, scale: u32) -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "amount", + Type::Primitive(PrimitiveType::Decimal { precision, scale }), + ) + .into(), + ]) + .build() + .unwrap() + } + + /// Decimal with precision <= 9 is stored as INT32 in Parquet. + /// The bloom filter contains i32 values (the unscaled mantissa). + #[test] + fn test_decimal_int32_present() { + let schema = create_decimal_schema(9, 2); + + // Parquet stores decimal(9,2) as INT32 with unscaled value + // Value "123.45" has mantissa 12345 + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i32); + sbbf.insert(&67890_i32); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT32, 0))]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(12345, 2), + 9, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Decimal INT32 value present should might-match"); + } + + #[test] + fn test_decimal_int32_absent() { + let schema = create_decimal_schema(9, 2); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i32); + sbbf.insert(&67890_i32); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT32, 0))]); + + // Value "999.99" has mantissa 99999, not in the bloom filter + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(99999, 2), + 9, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "Decimal INT32 value absent should not match"); + } + + /// Decimal with precision 10-18 is stored as INT64 in Parquet. + #[test] + fn test_decimal_int64_present() { + let schema = create_decimal_schema(15, 2); + + // "1234567890123.45" has mantissa 123456789012345 + let mantissa: i64 = 123456789012345; + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&mantissa); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0))]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa as i128, 2), + 15, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Decimal INT64 value present should might-match"); + } + + #[test] + fn test_decimal_int64_absent() { + let schema = create_decimal_schema(15, 2); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&123456789012345_i64); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0))]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(999999999999999, 2), + 15, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "Decimal INT64 value absent should not match"); + } + + /// Decimal with precision 19+ is stored as FIXED_LEN_BYTE_ARRAY in Parquet. + #[test] + fn test_decimal_fixed_bytes_present() { + let schema = create_decimal_schema(25, 2); + + // Large mantissa that requires FIXED_LEN_BYTE_ARRAY + let mantissa: i128 = 12345678901234567890; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 11).unwrap(); + let type_length = bytes.len() as i32; + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, type_length), + )]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa, 2), + 25, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Decimal FIXED_LEN_BYTE_ARRAY value present should might-match" + ); + } + + #[test] + fn test_decimal_fixed_bytes_absent() { + let schema = create_decimal_schema(25, 2); + + let mantissa: i128 = 12345678901234567890; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 11).unwrap(); + let type_length = bytes.len() as i32; + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, type_length), + )]); + + // Different value not in the bloom filter + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale( + 99999999999999999999, + 2, + ), + 25, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "Decimal FIXED_LEN_BYTE_ARRAY value absent should not match" + ); + } + + fn create_uuid_schema() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "u", Type::Primitive(PrimitiveType::Uuid)).into(), + ]) + .build() + .unwrap() + } + + /// The probe hands the 16 big-endian bytes to `check` as a slice while the writer + /// inserts them as a `ByteArray`; both hash the same bytes, so a present UUID must + /// still might-match. + #[test] + fn test_uuid_fixed_bytes_present() { + let uuid = uuid::Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap(); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(uuid.as_u128().to_be_bytes().to_vec())); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, 16), + )]); + + let predicate = Reference::new("u") + .equal_to(Datum::uuid(uuid)) + .bind(create_uuid_schema().into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "UUID present should might-match"); + } + + #[test] + fn test_uuid_fixed_bytes_absent() { + let present = uuid::Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap(); + let absent = uuid::Uuid::parse_str("00000000-0000-4000-8000-000000000001").unwrap(); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(present.as_u128().to_be_bytes().to_vec())); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, 16), + )]); + + let predicate = Reference::new("u") + .equal_to(Datum::uuid(absent)) + .bind(create_uuid_schema().into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "UUID absent should not match"); + } + + /// Negative decimal values should also work correctly. + #[test] + fn test_decimal_negative_int32() { + let schema = create_decimal_schema(9, 2); + + // "-123.45" has mantissa -12345 + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&(-12345_i32)); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT32, 0))]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(-12345, 2), + 9, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Negative decimal INT32 present should might-match"); + } + + #[test] + fn test_decimal_negative_fixed_bytes() { + let schema = create_decimal_schema(25, 2); + + let mantissa: i128 = -12345678901234567890; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 11).unwrap(); + let type_length = bytes.len() as i32; + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, type_length), + )]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa, 2), + 25, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Negative decimal FIXED_LEN_BYTE_ARRAY present should might-match" + ); + } + + /// A Parquet writer other than arrow-rs (e.g. Spark, Java Parquet) might + /// choose FIXED_LEN_BYTE_ARRAY for a low-precision decimal rather than INT32. + /// The evaluator must use the physical type from the file metadata, not assume + /// a mapping based on precision. + #[test] + fn test_decimal_low_precision_stored_as_fixed_len_byte_array() { + let schema = create_decimal_schema(5, 2); + + // Simulate a file where decimal(5,2) was stored as FIXED_LEN_BYTE_ARRAY + let mantissa: i128 = 12345; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 3).unwrap(); + let type_length = bytes.len() as i32; + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, type_length), + )]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa, 2), + 5, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Should match when physical type is FIXED_LEN_BYTE_ARRAY even for low-precision decimal" + ); + } + + /// Builds the (schema, bloom filter) pair for a `decimal` column that the file + /// stored at `file_type_length` bytes while the table schema now declares + /// `schema_precision`, as a precision-widening evolution produces. + fn widened_decimal_case( + mantissa: i128, + file_type_length: usize, + schema_precision: u32, + ) -> (Schema, HashMap) { + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, file_type_length).unwrap(); + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + sbbf, + PhysicalType::FIXED_LEN_BYTE_ARRAY, + file_type_length as i32, + ), + )]); + + (create_decimal_schema(schema_precision, 2), filters) + } + + fn decimal_eq_predicate(schema: Schema, mantissa: i128, precision: u32) -> BoundPredicate { + Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa, 2), + precision, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap() + } + + /// Widening a decimal's precision does not rewrite existing files, so the + /// column keeps its original `type_length`. Deriving the probe length from the + /// widened precision changed the encoding and missed every entry the writer + /// inserted, silently pruning row groups holding matching rows. + #[test] + fn test_decimal_widened_precision_present_is_not_pruned() { + let mantissa: i128 = 12345678901234567890; + // File written as decimal(20,2) -> 9 bytes; schema since widened to 38. + let (schema, filters) = widened_decimal_case(mantissa, 9, 38); + let predicate = decimal_eq_predicate(schema, mantissa, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!( + result, + "widened decimal precision must still find the value at the file's length" + ); + } + + #[test] + fn test_decimal_widened_precision_absent_still_prunes() { + let mantissa: i128 = 12345678901234567890; + let (schema, filters) = widened_decimal_case(mantissa, 9, 38); + let predicate = decimal_eq_predicate(schema, 999999999999999999, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!(!result, "genuinely absent value must still prune"); + } + + /// A value too wide for the column's `type_length` cannot be present, but + /// truncating it would probe an unrelated slot, so stay conservative. + #[test] + fn test_decimal_value_wider_than_column_does_not_prune() { + let (schema, filters) = widened_decimal_case(1234, 2, 38); + let predicate = decimal_eq_predicate(schema, i64::MAX as i128, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!(result, "value too wide for the column must not prune"); + } + + /// The `INT32` / `INT64` counterparts of the case above: after a precision + /// widening the bound literal can carry a mantissa wider than the file's + /// physical column, and a truncated probe would hash an unrelated slot. + #[test] + fn test_decimal_wider_than_int32_column_does_not_prune() { + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i32); + let filters = HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT32, 0))]); + + // File written as decimal(9,2) -> INT32; schema since widened to decimal(38,2). + let predicate = decimal_eq_predicate(create_decimal_schema(38, 2), i64::MAX as i128, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!( + result, + "mantissa too wide for an INT32 column must not prune" + ); + } + + #[test] + fn test_decimal_wider_than_int64_column_does_not_prune() { + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i64); + let filters = HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0))]); + + let predicate = + decimal_eq_predicate(create_decimal_schema(38, 2), i128::from(i64::MAX) + 1, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!( + result, + "mantissa too wide for an INT64 column must not prune" + ); + } + + /// A mantissa that does fit must still prune when genuinely absent, so the + /// `try_from` guard above does not disable pruning outright. + #[test] + fn test_decimal_within_int64_column_still_prunes() { + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i64); + let filters = HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0))]); + + let predicate = decimal_eq_predicate(create_decimal_schema(38, 2), 999_999, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!(!result, "absent in-range mantissa must still prune"); + } + + /// A zero or oversized `type_length` is unusable; never prune on it. + #[test] + fn test_decimal_unusable_type_length_does_not_prune() { + let mantissa: i128 = 1234; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 4).unwrap(); + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + for bogus_length in [0, -1, 17] { + let filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(), + PhysicalType::FIXED_LEN_BYTE_ARRAY, + bogus_length, + ), + )]); + let predicate = decimal_eq_predicate(create_decimal_schema(38, 2), mantissa, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!( + result, + "type_length {bogus_length} is unusable and must not prune" + ); + } + } + + fn single_field_schema(name: &str, ty: PrimitiveType) -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, name, Type::Primitive(ty)).into(), + ]) + .build() + .unwrap() + } + + /// After an `int` -> `long` promotion the predicate carries a `long`, but files + /// written before the promotion store the column as `INT32` and hashed it at + /// that width. Probing at the predicate's width made every lookup miss and + /// silently pruned row groups holding matching rows. + #[test] + fn test_promoted_int_to_long_present_is_not_pruned() { + let schema = single_field_schema("id", PrimitiveType::Long); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[100, 150, 199]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::long(150)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "long predicate against an INT32 column must find the promoted value" + ); + } + + #[test] + fn test_promoted_int_to_long_absent_still_prunes() { + let schema = single_field_schema("id", PrimitiveType::Long); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[100, 150, 199]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::long(4242)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "genuinely absent value must still prune"); + } + + /// A `long` predicate outside `i32` range cannot be present in an `INT32` + /// column; stay conservative rather than probe a truncated value. + #[test] + fn test_long_out_of_int32_range_does_not_prune() { + let schema = single_field_schema("id", PrimitiveType::Long); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::long(i64::from(i32::MAX) + 1)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "out-of-range long must not prune"); + } + + fn create_bloom_filter_with_values_f32(values: &[f32]) -> Sbbf { + let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64, 0.01).unwrap(); + for v in values { + sbbf.insert(v); + } + sbbf + } + + /// The `float` -> `double` counterpart of the `int` -> `long` promotion above. + #[test] + fn test_promoted_float_to_double_present_is_not_pruned() { + let schema = single_field_schema("val", PrimitiveType::Double); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_f32(&[1.5, 2.25, 4.0]), + PhysicalType::FLOAT, + 0, + ), + )]); + + let predicate = Reference::new("val") + .equal_to(Datum::double(1.5)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "double predicate against a FLOAT column must find the promoted value" + ); + } + + #[test] + fn test_promoted_float_to_double_absent_still_prunes() { + let schema = single_field_schema("val", PrimitiveType::Double); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_f32(&[1.5, 2.25, 4.0]), + PhysicalType::FLOAT, + 0, + ), + )]); + + let predicate = Reference::new("val") + .equal_to(Datum::double(9.75)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "genuinely absent value must still prune"); + } + + /// A double with no exact `f32` representation cannot equal any widened + /// `f32` in the column, so probing it would be meaningless; stay conservative. + #[test] + fn test_double_not_representable_as_f32_does_not_prune() { + let schema = single_field_schema("val", PrimitiveType::Double); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_f32(&[1.5, 2.25]), + PhysicalType::FLOAT, + 0, + ), + )]); + + let predicate = Reference::new("val") + .equal_to(Datum::double(0.1)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "non-representable double must not prune"); + } +} diff --git a/crates/iceberg/src/expr/visitors/mod.rs b/crates/iceberg/src/expr/visitors/mod.rs index 38332ea6a0..c5aca6838d 100644 --- a/crates/iceberg/src/expr/visitors/mod.rs +++ b/crates/iceberg/src/expr/visitors/mod.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +pub(crate) mod bloom_filter_evaluator; pub(crate) mod bound_predicate_visitor; pub(crate) mod expression_evaluator; pub(crate) mod inclusive_metrics_evaluator; diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 9bdf42beb8..081d64273d 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -142,6 +142,7 @@ pub struct TableScanBuilder<'a> { concurrency_limit_manifest_files: usize, row_group_filtering_enabled: bool, row_selection_enabled: bool, + bloom_filter_enabled: bool, } impl<'a> TableScanBuilder<'a> { @@ -160,6 +161,7 @@ impl<'a> TableScanBuilder<'a> { concurrency_limit_manifest_files: num_cpus, row_group_filtering_enabled: true, row_selection_enabled: false, + bloom_filter_enabled: false, } } @@ -266,6 +268,20 @@ impl<'a> TableScanBuilder<'a> { self } + /// Determines whether to enable bloom filter-based row group filtering. + /// + /// When enabled, if a read is performed with an equality or IN predicate, + /// the bloom filter for relevant columns in each row group is read and + /// checked. Row groups where the bloom filter proves the value is absent + /// are skipped entirely. + /// + /// Defaults to disabled, as reading bloom filters requires additional I/O + /// per column per row group. + pub fn with_bloom_filter_enabled(mut self, bloom_filter_enabled: bool) -> Self { + self.bloom_filter_enabled = bloom_filter_enabled; + self + } + /// Build the table scan. pub fn build(self) -> Result { let snapshot = match self.snapshot_id { @@ -292,6 +308,7 @@ impl<'a> TableScanBuilder<'a> { concurrency_limit_manifest_files: self.concurrency_limit_manifest_files, row_group_filtering_enabled: self.row_group_filtering_enabled, row_selection_enabled: self.row_selection_enabled, + bloom_filter_enabled: self.bloom_filter_enabled, runtime: self.table.runtime().clone(), }); }; @@ -349,6 +366,7 @@ impl<'a> TableScanBuilder<'a> { concurrency_limit_manifest_files: self.concurrency_limit_manifest_files, row_group_filtering_enabled: self.row_group_filtering_enabled, row_selection_enabled: self.row_selection_enabled, + bloom_filter_enabled: self.bloom_filter_enabled, runtime: self.table.runtime().clone(), }) } @@ -378,6 +396,7 @@ pub struct TableScan { row_group_filtering_enabled: bool, row_selection_enabled: bool, + bloom_filter_enabled: bool, runtime: Runtime, } @@ -510,7 +529,8 @@ impl TableScan { ArrowReaderBuilder::new(self.file_io.clone(), self.runtime.clone()) .with_data_file_concurrency_limit(self.concurrency_limit_data_files) .with_row_group_filtering_enabled(self.row_group_filtering_enabled) - .with_row_selection_enabled(self.row_selection_enabled); + .with_row_selection_enabled(self.row_selection_enabled) + .with_bloom_filter_enabled(self.bloom_filter_enabled); if let Some(batch_size) = self.batch_size { arrow_reader_builder = arrow_reader_builder.with_batch_size(batch_size); diff --git a/crates/iceberg/src/spec/values/decimal_utils.rs b/crates/iceberg/src/spec/values/decimal_utils.rs index 97225113cb..3d09d6f25e 100644 --- a/crates/iceberg/src/spec/values/decimal_utils.rs +++ b/crates/iceberg/src/spec/values/decimal_utils.rs @@ -196,6 +196,33 @@ pub fn i128_to_be_bytes_min(value: i128) -> Vec { bytes[start..].to_vec() } +/// Encode an i128 as exactly `len` big-endian two's complement bytes, matching a +/// Parquet `FIXED_LEN_BYTE_ARRAY` column of that declared `type_length`. +/// +/// Returns `None` if `value` does not fit in `len` bytes, since a truncated +/// encoding would represent a different number. +pub(crate) fn decimal_to_fixed_length_bytes_exact(value: i128, len: usize) -> Option> { + if len == 0 || len > 16 { + return None; + } + + let be_bytes = value.to_be_bytes(); + let offset = 16 - len; + let sign_byte = if value < 0 { 0xFF } else { 0x00 }; + + // The bytes being dropped must be pure sign extension. + if be_bytes[..offset].iter().any(|&b| b != sign_byte) { + return None; + } + + // The retained leading byte must carry the value's sign. + if (be_bytes[offset] & 0x80 != 0) != (value < 0) { + return None; + } + + Some(be_bytes[offset..].to_vec()) +} + #[cfg(test)] mod tests { use super::*; @@ -356,4 +383,77 @@ mod tests { ); } } + + #[test] + fn test_decimal_to_fixed_length_bytes_exact_positive() { + let bytes = decimal_to_fixed_length_bytes_exact(12345, 9).unwrap(); + assert_eq!(bytes.len(), 9); + // Big-endian, zero-padded on the left: 12345 = 0x3039 + assert_eq!(bytes[7], 0x30); + assert_eq!(bytes[8], 0x39); + assert!(bytes[..7].iter().all(|&b| b == 0x00)); + } + + #[test] + fn test_decimal_to_fixed_length_bytes_exact_negative() { + let bytes = decimal_to_fixed_length_bytes_exact(-1, 9).unwrap(); + assert_eq!(bytes.len(), 9); + assert!(bytes.iter().all(|&b| b == 0xFF)); + } + + #[test] + fn test_decimal_to_fixed_length_bytes_exact_round_trip() { + for value in [ + 0i128, + 1, + -1, + 12345, + -12345, + i64::MAX as i128, + i64::MIN as i128, + ] { + let bytes = decimal_to_fixed_length_bytes_exact(value, 16).unwrap(); + assert_eq!(bytes.len(), 16); + assert_eq!( + i128_from_be_bytes(&bytes), + Some(value), + "Round trip failed for value={value}" + ); + } + } + + /// The length must be honoured exactly: a value needing more bytes cannot be + /// truncated into the column's width, because the truncation would encode a + /// different number and probe the wrong bloom filter slot. + #[test] + fn test_decimal_to_fixed_length_bytes_exact_rejects_overflow() { + // 200 needs a leading zero byte to stay positive in two's complement. + assert_eq!(decimal_to_fixed_length_bytes_exact(200, 1), None); + assert_eq!( + decimal_to_fixed_length_bytes_exact(200, 2), + Some(vec![0x00, 0xC8]) + ); + + assert_eq!(decimal_to_fixed_length_bytes_exact(-200, 1), None); + assert_eq!( + decimal_to_fixed_length_bytes_exact(-200, 2), + Some(vec![0xFF, 0x38]) + ); + + // Exactly representable boundaries. + assert_eq!( + decimal_to_fixed_length_bytes_exact(127, 1), + Some(vec![0x7F]) + ); + assert_eq!(decimal_to_fixed_length_bytes_exact(128, 1), None); + assert_eq!( + decimal_to_fixed_length_bytes_exact(-128, 1), + Some(vec![0x80]) + ); + assert_eq!(decimal_to_fixed_length_bytes_exact(-129, 1), None); + + assert_eq!(decimal_to_fixed_length_bytes_exact(i128::MAX, 15), None); + assert_eq!(decimal_to_fixed_length_bytes_exact(0, 0), None); + assert_eq!(decimal_to_fixed_length_bytes_exact(0, 17), None); + } } From 72c0e05cf29c9386e70f43e199b39910591a5c6c Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 16 Sep 2026 12:08:44 +0200 Subject: [PATCH 71/74] fix(incremental-scan): expose bloom-filter row-group pruning on IncrementalTableScanBuilder The incremental-scan reader path already threads bloom_filter_enabled through apply_parquet_filters and both process_incremental_append_task/ process_equality_delete_task, but IncrementalTableScan::to_arrow()/ to_unzipped_arrow() never called ArrowReaderBuilder::with_bloom_filter_enabled, leaving it permanently disabled. Add a with_bloom_filter_enabled builder method on IncrementalTableScanBuilder, mirroring TableScanBuilder's, defaulting to disabled (matching upstream's conservative default given the per-row-group I/O cost) rather than silently forcing it on. Co-Authored-By: Claude Sonnet 5 --- crates/iceberg/public-api.txt | 1 + crates/iceberg/src/scan/incremental/mod.rs | 25 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 55f3eba70e..eb50444164 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1659,6 +1659,7 @@ pub fn iceberg::scan::incremental::IncrementalTableScanBuilder<'a>::select_all(s pub fn iceberg::scan::incremental::IncrementalTableScanBuilder<'a>::select_empty(self) -> Self pub fn iceberg::scan::incremental::IncrementalTableScanBuilder<'a>::to_snapshot_id(self, to_snapshot_id: i64) -> Self pub fn iceberg::scan::incremental::IncrementalTableScanBuilder<'a>::with_batch_size(self, batch_size: core::option::Option) -> Self +pub fn iceberg::scan::incremental::IncrementalTableScanBuilder<'a>::with_bloom_filter_enabled(self, bloom_filter_enabled: bool) -> Self pub fn iceberg::scan::incremental::IncrementalTableScanBuilder<'a>::with_case_sensitive(self, case_sensitive: bool) -> Self pub fn iceberg::scan::incremental::IncrementalTableScanBuilder<'a>::with_data_file_concurrency_limit(self, limit: usize) -> Self pub fn iceberg::scan::incremental::IncrementalTableScanBuilder<'a>::with_file_column(self) -> Self diff --git a/crates/iceberg/src/scan/incremental/mod.rs b/crates/iceberg/src/scan/incremental/mod.rs index afe08a187f..90de4feb14 100644 --- a/crates/iceberg/src/scan/incremental/mod.rs +++ b/crates/iceberg/src/scan/incremental/mod.rs @@ -71,6 +71,7 @@ pub struct IncrementalTableScanBuilder<'a> { concurrency_limit_data_files: usize, concurrency_limit_manifest_entries: usize, concurrency_limit_manifest_files: usize, + bloom_filter_enabled: bool, } impl<'a> IncrementalTableScanBuilder<'a> { @@ -90,6 +91,7 @@ impl<'a> IncrementalTableScanBuilder<'a> { concurrency_limit_data_files: num_cpus, concurrency_limit_manifest_entries: num_cpus, concurrency_limit_manifest_files: num_cpus, + bloom_filter_enabled: false, } } @@ -239,6 +241,21 @@ impl<'a> IncrementalTableScanBuilder<'a> { self } + /// Determines whether to enable bloom filter-based row group filtering. + /// + /// When enabled, if a read is performed with an equality or IN predicate, + /// the bloom filter for relevant columns in each row group is read and + /// checked. Row groups where the bloom filter proves the value is absent + /// are skipped entirely. + /// + /// Defaults to disabled. Each bloom filter is a separate read, and they are + /// issued serially — one round trip per relevant column per row group, before + /// any data is read. + pub fn with_bloom_filter_enabled(mut self, bloom_filter_enabled: bool) -> Self { + self.bloom_filter_enabled = bloom_filter_enabled; + self + } + /// Build the incremental table scan. pub fn build(self) -> Result { let metadata = self.table.metadata(); @@ -429,6 +446,7 @@ impl<'a> IncrementalTableScanBuilder<'a> { concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries, concurrency_limit_manifest_files: self.concurrency_limit_manifest_files, runtime: self.table.runtime().clone(), + bloom_filter_enabled: self.bloom_filter_enabled, }) } } @@ -444,6 +462,7 @@ pub struct IncrementalTableScan { concurrency_limit_manifest_entries: usize, concurrency_limit_manifest_files: usize, runtime: Runtime, + bloom_filter_enabled: bool, } impl IncrementalTableScan { @@ -807,7 +826,8 @@ impl IncrementalTableScan { ArrowReaderBuilder::new(self.file_io.clone(), self.runtime.clone()) .with_data_file_concurrency_limit(self.concurrency_limit_data_files) .with_row_group_filtering_enabled(true) - .with_row_selection_enabled(true); + .with_row_selection_enabled(true) + .with_bloom_filter_enabled(self.bloom_filter_enabled); if let Some(batch_size) = self.batch_size { arrow_reader_builder = arrow_reader_builder.with_batch_size(batch_size); @@ -826,7 +846,8 @@ impl IncrementalTableScan { ArrowReaderBuilder::new(self.file_io.clone(), self.runtime.clone()) .with_data_file_concurrency_limit(self.concurrency_limit_data_files) .with_row_group_filtering_enabled(true) - .with_row_selection_enabled(true); + .with_row_selection_enabled(true) + .with_bloom_filter_enabled(self.bloom_filter_enabled); if let Some(batch_size) = self.batch_size { arrow_reader_builder = arrow_reader_builder.with_batch_size(batch_size); From b5bb309c515441c52534fd1781d7caa4cc555110 Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 16 Sep 2026 12:35:45 +0200 Subject: [PATCH 72/74] fix(ci): drop crates.io version pin from iceberg-storage-refreshable workspace dep crates/storage/refreshable is publish = false (fork-only, never published to crates.io), but the root workspace dependency table gave it a version like every other (published) workspace crate. iceberg-catalog-rest depends on it as a dev-dependency via workspace = true, so cargo publish tried to resolve "iceberg-storage-refreshable = 0.10.0" from the registry and failed ("no matching package named `iceberg-storage-refreshable` found"). Dropping the version leaves it path-only, matching the existing pattern used for iceberg_test_utils (another unpublished dev-only workspace crate), which cargo correctly omits from the published manifest. Co-Authored-By: Claude Sonnet 5 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 96d93e669d..51f0340690 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,7 +108,7 @@ iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } iceberg-datafusion = { version = "0.10.0", path = "./crates/integrations/datafusion" } iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } -iceberg-storage-refreshable = { version = "0.10.0", path = "./crates/storage/refreshable" } +iceberg-storage-refreshable = { path = "./crates/storage/refreshable" } indicatif = "0.18" itertools = "0.13" libtest-mimic = "0.8.1" From 5d81656fbcb775887755135af2e5dd22ca7fec5c Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 16 Sep 2026 12:35:54 +0200 Subject: [PATCH 73/74] fix(scan): tolerate a dropped partition source column in FileScanTask::validate() Upstream's new FileScanTask::validate() unconditionally calls partition_spec.partition_type(&schema), which errors when a historical partition spec references a source column no longer present in the current schema -- a legitimate v2+ state the fork already tolerates one layer up in PartitionFilterCache::get (falls back to an always-true partition filter rather than erroring). validate() didn't know about that fallback, so it silently regressed a fork capability: filtered scans over a table that evolved past a historical spec's source column now hard-failed instead of falling back gracefully, and the merge's conflict resolution had updated the regression test to assert the new error rather than catching the incompatibility. Mirror PartitionFilterCache::get's own dropped-source-column detection in validate() and skip the partition_type() check in that case. Restores scan::mod::test_filtered_scan_with_dropped_partition_source_column to its original success-path assertions, and updates the new upstream unit test (scan::task::tests) to expect success instead of rejection for the same scenario. Co-Authored-By: Claude Sonnet 5 --- crates/iceberg/src/scan/mod.rs | 35 +++++++++++++++++++++++++-------- crates/iceberg/src/scan/task.rs | 29 +++++++++++++++++++++------ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index f95f42a0d4..e60bef9e9e 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -2373,10 +2373,15 @@ pub mod tests { } #[tokio::test] - async fn test_filtered_scan_rejects_dropped_partition_source_column() { + async fn test_filtered_scan_with_dropped_partition_source_column() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; + // baseline: the same filtered scan against the table before evolution + let baseline = scan_y_gte_5(&fixture.table).await; + assert!(!baseline.is_empty()); + assert!(baseline.iter().all(|y| *y >= 5)); + // Evolve the table so that the manifests reference a historical spec whose source // column is no longer in the current schema: make an unpartitioned spec the // default, then drop the original spec's source column from the schema. @@ -2426,23 +2431,37 @@ pub mod tests { .metadata; let table = fixture.table.clone().with_metadata(Arc::new(metadata)); + // planning and reading must succeed, and the results must match the table before + // evolution: no rows wrongly pruned and none returned unfiltered + let evolved = scan_y_gte_5(&table).await; + assert_eq!(evolved, baseline); + } + + async fn scan_y_gte_5(table: &Table) -> Vec { let table_scan = table .scan() .select(["y"]) .with_filter(Reference::new("y").greater_than_or_equal_to(Datum::long(5))) .build() .unwrap(); - - let err = table_scan - .plan_files() + let batches: Vec<_> = table_scan + .to_arrow() .await .unwrap() - .try_collect::>() + .try_collect() .await - .unwrap_err(); + .unwrap(); - assert_eq!(err.kind(), ErrorKind::Unexpected); - assert!(err.message().contains("No column with source column id 1")); + let mut values: Vec = batches + .iter() + .flat_map(|batch| { + let col = batch.column_by_name("y").unwrap(); + let arr = col.as_any().downcast_ref::().unwrap(); + (0..arr.len()).map(|i| arr.value(i)).collect::>() + }) + .collect(); + values.sort_unstable(); + values } #[tokio::test] diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 5fd0f81bb6..b5f4cef9bb 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -283,7 +283,18 @@ impl FileScanTask { )) } (Some(_), Some(partition_spec)) => { - partition_spec.partition_type(&self.schema)?; + // A historical spec may reference a source column that was later dropped + // from the schema, which is a legitimate v2+ state (see + // `PartitionFilterCache::get`, which applies the same fallback for the same + // reason). Such a spec cannot be resolved to a partition type; skip this + // check rather than rejecting an otherwise-valid task. + let has_dropped_source_column = partition_spec + .fields() + .iter() + .any(|field| self.schema.field_by_id(field.source_id).is_none()); + if !has_dropped_source_column { + partition_spec.partition_type(&self.schema)?; + } Ok(()) } } @@ -689,7 +700,11 @@ mod tests { } #[test] - fn test_file_scan_task_builder_rejects_dropped_partition_source_column() { + fn test_file_scan_task_builder_tolerates_dropped_partition_source_column() { + // A historical spec may reference a source column that was later dropped from the + // schema, which is a legitimate v2+ state (see `PartitionFilterCache::get`, which + // applies the same tolerance at the partition-filter layer for the same reason). + // Building the task must succeed rather than reject it. let (_historical_schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity); let current_schema = Arc::new( @@ -703,15 +718,17 @@ mod tests { .unwrap(), ); - let err = build_file_scan_task( + let task = build_file_scan_task( current_schema, Some(Struct::from_iter([Some(Literal::long(42))])), Some(partition_spec), ) - .unwrap_err(); + .unwrap(); - assert_eq!(err.kind(), ErrorKind::Unexpected); - assert!(err.message().contains("No column with source column id 1")); + assert_eq!( + task.partition(), + Some(&Struct::from_iter([Some(Literal::long(42))])) + ); } #[test] From 2c6aef66ebdfd475266cc23b2576d0fa8cf37ca7 Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Wed, 16 Sep 2026 10:51:40 +0200 Subject: [PATCH 74/74] fix(transaction): fix correctness bugs in OverwriteAction Prompted by review comments on the upstream PR our OverwriteAction was originally built on top of (apache/iceberg-rust#2185). Checked each comment against our own implementation; three applied here too: - Resurrection bug (the serious one): rewrite_manifest's fallback branch called add_existing_entry for every entry not newly deleted this round, including entries that were already Deleted by a prior overwrite -- add_existing_entry unconditionally resets status to Existing, silently resurrecting previously-deleted files as live data the next time their manifest got rewritten for an unrelated reason. Fixed by routing already-non-alive entries through add_deleted_entry instead, preserving their original snapshot_id as a tombstone. Added test_second_overwrite_does_not_resurrect_deleted_file to lock this in. - Wrong schema-id on rewritten manifests: schema was taken from table.metadata().current_schema() (the table's *current* schema) rather than the manifest being rewritten's own schema. After any schema evolution between the original write and a later overwrite, this stamped the wrong schema-id on old entries, which schema-id-aware readers (Java, PyIceberg) would misinterpret. Fixed by deriving both schema and partition spec directly from the manifest's own ManifestMetadata (which already carries fully resolved objects, not just IDs) -- this is also simpler than the existing partition_spec_by_id table lookup it replaces, and doesn't depend on the table still listing that partition spec. - Manifest naming/location bypassed convention: rewritten manifests were written to a hardcoded `{location}/metadata/` path with a fresh random UUID per manifest, rather than `metadata_location()` (which respects a configured write.metadata.path table property, already used correctly elsewhere in this same file for the manifest list) and the commit's own UUID (shared across every manifest touched by one commit, matching SnapshotProducer::new_manifest_writer's own convention). Added a commit_uuid() getter on SnapshotProducer and an index parameter to rewrite_manifest to keep names unique when a commit rewrites more than one manifest. Also fixed, one level up in shared code: SnapshotProducer::summary() unconditionally applied "truncate full table" semantics (replace computed added/removed counts with the previous snapshot's totals) for any Operation::Overwrite, which is wrong for OverwriteAction's explicit-file-list partial overwrites -- it already knows exactly what was added/removed. Added a `truncate_full_table()` method to SnapshotProduceOperation (default false, only meaningful for an operation that genuinely replaces the whole table) and had OverwriteOperation opt out explicitly. Updated test_delete_only_overwrite_summary, which had documented the old (wrong) truncated counts as expected behavior, to assert the correct ones. Not applicable: the review's manifest-filtering concern (dropping delete-only manifests) is already handled correctly on our side -- both FastAppendAction and OverwriteAction's existing_manifest() were confirmed to already keep manifests with has_deleted_files(). Co-Authored-By: Claude Sonnet 5 --- crates/iceberg/src/transaction/overwrite.rs | 145 +++++++++++++++----- crates/iceberg/src/transaction/snapshot.rs | 17 ++- 2 files changed, 130 insertions(+), 32 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 68724bc0b5..051c753753 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -149,6 +149,13 @@ impl SnapshotProduceOperation for OverwriteOperation { Operation::Overwrite } + // `OverwriteAction` always works from an explicit, exact list of added/deleted data + // files, so the snapshot summary's computed added/removed counts are already correct + // -- never truncate them to "everything in the previous snapshot was removed." + fn truncate_full_table(&self) -> bool { + false + } + async fn delete_entries( &self, _snapshot_produce: &SnapshotProducer<'_>, @@ -184,6 +191,10 @@ impl SnapshotProduceOperation for OverwriteOperation { } let mut result = Vec::new(); + // Shared across every manifest rewritten by this one commit, so rewritten + // manifest names stay unique within the commit without needing a fresh random + // UUID per manifest (see `rewrite_manifest`). + let mut rewrite_counter: u64 = 0; for manifest_file in manifest_list.entries() { if !manifest_file.has_added_files() @@ -205,8 +216,9 @@ impl SnapshotProduceOperation for OverwriteOperation { if has_deletes { let rewritten = self - .rewrite_manifest(snapshot_produce, manifest_file, &manifest) + .rewrite_manifest(snapshot_produce, manifest_file, &manifest, rewrite_counter) .await?; + rewrite_counter += 1; result.push(rewritten); } else { result.push(manifest_file.clone()); @@ -219,36 +231,37 @@ impl SnapshotProduceOperation for OverwriteOperation { impl OverwriteOperation { /// Rewrite a manifest, marking entries whose file paths are in `deleted_file_paths` - /// as `ManifestStatus::Deleted`. + /// as `ManifestStatus::Deleted`. `index` disambiguates this manifest's name from + /// others rewritten by the same commit (see `rewrite_counter` in `existing_manifest`). async fn rewrite_manifest( &self, snapshot_produce: &SnapshotProducer<'_>, manifest_file: &ManifestFile, manifest: &crate::spec::Manifest, + index: u64, ) -> Result { let table = snapshot_produce.table; + // Match the naming/location convention `SnapshotProducer::new_manifest_writer` + // uses for newly-written manifests: `metadata_location()` (which, unlike a bare + // `location()`, respects a configured `write.metadata.path`) and the commit's own + // UUID, so every manifest touched by one commit is identifiable by that UUID + // rather than each rewritten manifest getting its own unrelated random one. let new_manifest_path = format!( - "{}/metadata/{}-m-overwrite.avro", - table.metadata().location(), - Uuid::now_v7(), + "{}/{}-m-overwrite-{}.avro", + table.metadata().metadata_location()?, + snapshot_produce.commit_uuid(), + index, ); let output_file = table.file_io().new_output(&new_manifest_path)?; - let partition_spec = table - .metadata() - .partition_spec_by_id(manifest_file.partition_spec_id) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Partition spec {} not found in table metadata", - manifest_file.partition_spec_id - ), - ) - })? - .as_ref() - .clone(); - let schema = table.metadata().current_schema().clone(); + // Derive schema and partition spec from the manifest being rewritten itself, + // not the table's current/default ones: if the table has since undergone schema + // or partition evolution, stamping the current schema-id/partition-spec-id onto + // entries written under an older one would misrepresent them and cause + // implementations that respect the manifest's own schema-id (e.g. Java, + // PyIceberg) to misread them. + let partition_spec = manifest.metadata().partition_spec().clone(); + let schema = manifest.metadata().schema().clone(); // Preserve the original manifest's own encryption key when rewriting it, rather // than generating a new one, so the rewritten manifest stays decryptable the same @@ -291,6 +304,14 @@ impl OverwriteOperation { let mut deleted: ManifestEntry = (**entry).clone(); deleted.snapshot_id = Some(self.snapshot_id); writer.add_deleted_entry(deleted)?; + } else if !entry.is_alive() { + // Already deleted by a prior snapshot. Preserve it as a tombstone via + // add_deleted_entry (which keeps its original snapshot_id, since we + // don't touch it here) rather than add_existing_entry, which + // unconditionally resets status to Existing and would resurrect this + // file as live data. + let cloned: ManifestEntry = (**entry).clone(); + writer.add_deleted_entry(cloned)?; } else { let cloned: ManifestEntry = (**entry).clone(); writer.add_existing_entry(cloned)?; @@ -705,23 +726,85 @@ mod tests { let props = &snapshot.summary().additional_properties; - // Overwrite semantics use truncate_table_summary which treats overwrite as - // "replace all": deleted-data-files reflects the previous total (3), not the - // number of explicitly deleted files (1). Without Fix 1, previous_snapshot - // would be None so previous_total=0 and u64 subtraction would underflow. + // OverwriteAction always works from an explicit, exact file list, so a partial + // (delete-only) overwrite must report exactly what was removed (1), not the + // previous snapshot's total (3) -- `truncate_full_table()` returning `false` + // for OverwriteOperation is what keeps `update_snapshot_summaries` from + // overwriting these already-correct computed counts with "replace all" + // semantics. assert_eq!( props.get("deleted-data-files").map(|s| s.as_str()), - Some("3"), - "Expected deleted-data-files=3 (overwrite semantics), got: {props:?}" + Some("1"), + "Expected deleted-data-files=1 (only the explicitly deleted file), got: {props:?}" ); - // total-data-files should be 0 after a delete-only overwrite (overwrite - // semantics: 3 removed - 3 previous = 0). Without Fix 1 this would panic - // with u64 underflow in debug mode. + // total-data-files should be 2 after deleting 1 of the 3 previously-appended files. assert_eq!( props.get("total-data-files").map(|s| s.as_str()), - Some("0"), - "Expected total-data-files=0 (overwrite semantics), got: {props:?}" + Some("2"), + "Expected total-data-files=2 (3 appended - 1 deleted), got: {props:?}" + ); + } + + #[tokio::test] + async fn test_second_overwrite_does_not_resurrect_deleted_file() { + use crate::memory::tests::new_memory_catalog; + use crate::transaction::ApplyTransactionAction; + use crate::transaction::tests::make_v3_minimal_table_in_catalog; + + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + + // Append files A and B together so they land in the same manifest. + let file_a = test_data_file("test/a.parquet", spec_id); + let file_b = test_data_file("test/b.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .add_data_files(vec![file_a.clone(), file_b.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // First overwrite: delete A. Rewrites the shared manifest: A -> Deleted, B -> + // Existing (carried forward via add_existing_entry). + let tx = Transaction::new(&table); + let action = tx.overwrite().delete_data_files(vec![file_a.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Second overwrite: delete B. This forces the *same* manifest (which already + // holds A as a Deleted tombstone) to be rewritten again. Before the fix, A's + // entry would fall into rewrite_manifest's `else` branch and go through + // add_existing_entry, which unconditionally resets status to Existing -- + // resurrecting A as live data. + let tx = Transaction::new(&table); + let action = tx.overwrite().delete_data_files(vec![file_b.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + + let mut all_entries = vec![]; + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + for entry in manifest.entries() { + all_entries.push((entry.status(), entry.file_path().to_string())); + } + } + + assert!( + all_entries.iter().any( + |(status, path)| *status == ManifestStatus::Deleted && path == "test/a.parquet" + ), + "File A must still be Deleted after the second overwrite, not resurrected: {all_entries:?}", + ); + assert!( + all_entries.iter().any( + |(status, path)| *status == ManifestStatus::Deleted && path == "test/b.parquet" + ), + "File B must be Deleted after being targeted by the second overwrite: {all_entries:?}", ); } } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index aa9cf3818a..9292258e71 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -66,6 +66,17 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { /// which is stored in the snapshot metadata for tracking and auditing purposes. fn operation(&self) -> Operation; + /// Whether the snapshot summary should report a full-table truncate: computed + /// added/removed counts are discarded and every `total-*` property is replaced with + /// the previous snapshot's totals under `deleted-*`/`removed-*` (see + /// `truncate_table_summary`). This only makes sense for an operation that genuinely + /// replaces the entire table's contents; an operation with an explicit, partial set + /// of added/deleted files already has exact counts and must not have them overwritten + /// with the previous snapshot's totals. Defaults to `false`. + fn truncate_full_table(&self) -> bool { + false + } + /// Returns manifest entries that should be marked as deleted in the new snapshot. #[allow(unused)] fn delete_entries( @@ -125,6 +136,10 @@ impl<'a> SnapshotProducer<'a> { self.snapshot_id } + pub(crate) fn commit_uuid(&self) -> Uuid { + self.commit_uuid + } + pub(crate) fn new( table: &'a Table, commit_uuid: Uuid, @@ -442,7 +457,7 @@ impl<'a> SnapshotProducer<'a> { update_snapshot_summaries( summary, previous_snapshot.map(|s| s.summary()), - snapshot_produce_operation.operation() == Operation::Overwrite, + snapshot_produce_operation.truncate_full_table(), ) }