|
63 | 63 | BoundNotStartsWith, |
64 | 64 | BoundReference, |
65 | 65 | BoundStartsWith, |
| 66 | + EqualTo, |
66 | 67 | GreaterThan, |
| 68 | + IsNull, |
67 | 69 | Not, |
68 | 70 | Or, |
69 | 71 | ) |
@@ -1495,6 +1497,42 @@ def test_identity_transform_column_projection_with_falsy_value( |
1495 | 1497 | ) |
1496 | 1498 |
|
1497 | 1499 |
|
| 1500 | +def test_identity_transform_null_partition_value_takes_precedence_over_initial_default(tmpdir: str) -> None: |
| 1501 | + """A null identity partition value must project as null, not as the field's initial-default.""" |
| 1502 | + table_schema = Schema( |
| 1503 | + NestedField(1, "other_field", StringType(), required=False), |
| 1504 | + NestedField(2, "partition_col", IntegerType(), required=False, initial_default=7), |
| 1505 | + ) |
| 1506 | + partition_spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "partition_col")) |
| 1507 | + table_metadata = TableMetadataV3( |
| 1508 | + location=str(tmpdir), last_column_id=2, schemas=[table_schema], partition_specs=[partition_spec], next_row_id=0 |
| 1509 | + ) |
| 1510 | + |
| 1511 | + arrow_schema = pa.schema([pa.field("other_field", pa.string(), metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"})]) |
| 1512 | + file_path = _write_table_to_file( |
| 1513 | + f"{tmpdir}/null_partition.parquet", arrow_schema, pa.table({"other_field": ["foo", "bar"]}, schema=arrow_schema) |
| 1514 | + ) |
| 1515 | + data_file = DataFile.from_args( |
| 1516 | + content=DataFileContent.DATA, |
| 1517 | + file_path=file_path, |
| 1518 | + file_format=FileFormat.PARQUET, |
| 1519 | + partition=Record(None), |
| 1520 | + record_count=2, |
| 1521 | + file_size_in_bytes=os.path.getsize(file_path), |
| 1522 | + spec_id=partition_spec.spec_id, |
| 1523 | + ) |
| 1524 | + |
| 1525 | + def scan(projected_schema: Schema, row_filter: BooleanExpression) -> pa.Table: |
| 1526 | + return ArrowScan(table_metadata, PyArrowFileIO(), projected_schema, row_filter).to_table([FileScanTask(data_file)]) |
| 1527 | + |
| 1528 | + assert scan(table_schema, AlwaysTrue()).to_pydict() == {"other_field": ["foo", "bar"], "partition_col": [None, None]} |
| 1529 | + |
| 1530 | + # Filters on the partition column see null whether or not the column is projected |
| 1531 | + for projected_schema in (table_schema, Schema(table_schema.fields[0])): |
| 1532 | + assert scan(projected_schema, IsNull("partition_col"))["other_field"].to_pylist() == ["foo", "bar"] |
| 1533 | + assert scan(projected_schema, EqualTo("partition_col", 7)).num_rows == 0 |
| 1534 | + |
| 1535 | + |
1498 | 1536 | @pytest.mark.parametrize("with_field_ids", [False, True], ids=["name-mapping", "field-ids"]) |
1499 | 1537 | def test_identity_transform_columns_projection(tmp_path: str, catalog: InMemoryCatalog, with_field_ids: bool) -> None: |
1500 | 1538 | # Test by adding a non-partitioned data file to a multi-partitioned table, verifying partition value |
|
0 commit comments