diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index 92b1dda11..19de6ac2a 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -194,6 +194,11 @@ and `Arrow DataTypes `` - List - Data type of an array of elements with same subtype. diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 7717b27fb..63801ba97 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -1002,19 +1002,14 @@ Status SchemaValidation::ValidateLanceDataField(const std::shared_ptr(*type); - if (!timestamp_type.timezone().empty()) { - return Status::Invalid( - "Lance file format does not support type LOCAL_ZONED_TIMESTAMP"); - } - return Status::OK(); - } case arrow::Type::LIST: return ValidateLanceDataField(type->field(0)); case arrow::Type::STRUCT: diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 4f7b41160..2b9c6957f 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -439,6 +439,14 @@ TEST(SchemaValidationTest, TestLanceDataTypes) { arrow::field("date", arrow::date32()), arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI)), arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::MICRO)), + arrow::field("ltz_0", arrow::timestamp(arrow::TimeUnit::SECOND, "UTC")), + arrow::field("ltz_3", arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai")), + arrow::field("ltz_6", arrow::timestamp(arrow::TimeUnit::MICRO, "GMT-10:00")), + arrow::field("ltz_9", arrow::timestamp(arrow::TimeUnit::NANO, "+08:00")), + arrow::field("ltz_array", arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"))), + arrow::field("nested_ltz", + arrow::struct_({arrow::field( + "values", arrow::list(arrow::timestamp(arrow::TimeUnit::MILLI, "UTC")))})), arrow::field("decimal", arrow::decimal128(38, 2)), arrow::field("array", arrow::list(arrow::float32())), arrow::field("nested_time", @@ -474,11 +482,9 @@ TEST(SchemaValidationTest, TestLanceDataTypes) { arrow::FieldVector unsupported_fields = { arrow::field("map", arrow::map(arrow::int32(), arrow::utf8())), - arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")), VariantTypeUtils::ToArrowField("variant"), }; - std::vector expected_errors = {"type MAP", "LOCAL_ZONED_TIMESTAMP", - "type VARIANT"}; + std::vector expected_errors = {"type MAP", "type VARIANT"}; for (size_t i = 0; i < unsupported_fields.size(); ++i) { ASSERT_OK_AND_ASSIGN( table_schema, @@ -504,6 +510,14 @@ TEST(SchemaValidationTest, TestLanceDataTypes) { {Options::CHANGELOG_FILE_FORMAT, "lance"}}) { std::map alternate_format_options = { {Options::BUCKET, "-1"}, {Options::FILE_FORMAT, "parquet"}, {option_key, option_value}}; + ASSERT_OK_AND_ASSIGN( + table_schema, + TableSchema::Create( + /*schema_id=*/0, + arrow::schema({arrow::field( + "ltz", arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, "GMT-10:00")))}), + /*partition_keys=*/{}, /*primary_keys=*/{}, alternate_format_options)); + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); ASSERT_OK_AND_ASSIGN( table_schema, TableSchema::Create( diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index ff6368a7e..ff0690292 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -727,6 +727,92 @@ std::shared_ptr NormalizeLeafRepresentation( } return t; } + +std::shared_ptr AlignTimestampTimezonesData( + const std::shared_ptr& data, + const std::shared_ptr& read_type) { + const std::shared_ptr& type = data->type; + if (type->id() == arrow::Type::TIMESTAMP && read_type->id() == arrow::Type::TIMESTAMP) { + const auto& stored = checked_cast(*type); + const auto& wanted = checked_cast(*read_type); + if (stored.unit() != wanted.unit() || stored.timezone().empty() || + wanted.timezone().empty() || stored.timezone() == wanted.timezone()) { + return data; + } + std::shared_ptr retyped = data->Copy(); + retyped->type = arrow::timestamp(wanted.unit(), wanted.timezone()); + return retyped; + } + if (type->id() == arrow::Type::STRUCT && read_type->id() == arrow::Type::STRUCT) { + const auto& wanted_struct = checked_cast(*read_type); + arrow::FieldVector fields = type->fields(); + std::shared_ptr retyped; + for (int32_t i = 0; i < static_cast(data->child_data.size()); ++i) { + const int32_t index = wanted_struct.GetFieldIndex(fields[i]->name()); + if (index < 0) { + continue; + } + std::shared_ptr child = AlignTimestampTimezonesData( + data->child_data[i], wanted_struct.field(index)->type()); + if (child == data->child_data[i]) { + continue; + } + if (retyped == nullptr) { + retyped = data->Copy(); + } + retyped->child_data[i] = child; + fields[i] = fields[i]->WithType(child->type); + } + if (retyped == nullptr) { + return data; + } + retyped->type = arrow::struct_(fields); + return retyped; + } + if ((type->id() == arrow::Type::LIST || type->id() == arrow::Type::FIXED_SIZE_LIST) && + type->id() == read_type->id() && !data->child_data.empty()) { + std::shared_ptr values = + AlignTimestampTimezonesData(data->child_data[0], read_type->field(0)->type()); + if (values == data->child_data[0]) { + return data; + } + std::shared_ptr retyped = data->Copy(); + retyped->child_data[0] = values; + const std::shared_ptr value_field = type->field(0)->WithType(values->type); + retyped->type = type->id() == arrow::Type::LIST + ? arrow::list(value_field) + : arrow::fixed_size_list( + value_field, + checked_cast(*type).list_size()); + return retyped; + } + if (type->id() == arrow::Type::MAP && read_type->id() == arrow::Type::MAP && + !data->child_data.empty()) { + const auto& stored_map = checked_cast(*type); + const auto& wanted_map = checked_cast(*read_type); + const std::shared_ptr& entries = data->child_data[0]; + std::shared_ptr keys = + AlignTimestampTimezonesData(entries->child_data[0], wanted_map.key_type()); + std::shared_ptr items = + AlignTimestampTimezonesData(entries->child_data[1], wanted_map.item_type()); + if (keys == entries->child_data[0] && items == entries->child_data[1]) { + return data; + } + const std::shared_ptr key_field = + stored_map.key_field()->WithType(keys->type); + const std::shared_ptr item_field = + stored_map.item_field()->WithType(items->type); + std::shared_ptr retyped_entries = entries->Copy(); + retyped_entries->type = arrow::struct_({key_field, item_field}); + retyped_entries->child_data = {keys, items}; + std::shared_ptr retyped = data->Copy(); + retyped->type = + std::make_shared(key_field, item_field, stored_map.keys_sorted()); + retyped->child_data = {retyped_entries}; + return retyped; + } + return data; +} } // namespace Result> NestedProjectionUtils::AlignArrayToReadType( @@ -840,4 +926,13 @@ Result> NestedProjectionUtils::AlignArrayToReadTyp } } +std::shared_ptr NestedProjectionUtils::AlignTimestampTimezones( + const std::shared_ptr& array, const std::shared_ptr& read_type) { + std::shared_ptr data = AlignTimestampTimezonesData(array->data(), read_type); + if (data == array->data()) { + return array; + } + return arrow::MakeArray(data); +} + } // namespace paimon diff --git a/src/paimon/core/utils/nested_projection_utils.h b/src/paimon/core/utils/nested_projection_utils.h index 083e878e3..b25aef5dd 100644 --- a/src/paimon/core/utils/nested_projection_utils.h +++ b/src/paimon/core/utils/nested_projection_utils.h @@ -128,6 +128,16 @@ class PAIMON_EXPORT NestedProjectionUtils { const std::shared_ptr& array, const std::shared_ptr& read_type, arrow::MemoryPool* pool); + /// Relabel a TIMESTAMP in `array` with the zone `read_type` asks for, only when both sides + /// have a zone and the same unit. A zoned value is an instant since the epoch, so only the + /// type changes. STRUCT children are matched by name, and LIST/FIXED_SIZE_LIST elements and + /// MAP keys and values are followed; every other difference, a unit change included, is left + /// as is. + /// @return `array` itself when no TIMESTAMP is relabelled, otherwise a shallow copy. + static std::shared_ptr AlignTimestampTimezones( + const std::shared_ptr& array, + const std::shared_ptr& read_type); + private: static Result HasNestedSubfieldProjectionType( const std::shared_ptr& file_type, diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index e052c521f..9ca69e91e 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -520,6 +520,94 @@ TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsNestedLargeBinaryBlob) ASSERT_TRUE(blob_out->Equals(*blob)); } +TEST(NestedProjectionUtilsTest, AlignTimestampTimezonesRelabelsZonedLeavesOfSameUnit) { + auto make_type = [](const std::string& zone, arrow::TimeUnit::type other_unit) { + return arrow::struct_({ + arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MICRO, zone)), + arrow::field("row", arrow::struct_({arrow::field( + "ltz", arrow::timestamp(arrow::TimeUnit::MILLI, zone))})), + arrow::field("list", arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, zone))), + arrow::field("fixed_list", + arrow::fixed_size_list(arrow::timestamp(arrow::TimeUnit::MICRO, zone), 2)), + arrow::field("map", arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI, zone), + arrow::timestamp(arrow::TimeUnit::MICRO, zone))), + arrow::field("other_unit", arrow::timestamp(other_unit, zone)), + arrow::field("ntz", arrow::timestamp(arrow::TimeUnit::MICRO)), + }); + }; + const std::string json = R"([[1, [2], [3, null], [16, null], [[11, 12], [13, null]], 4, 5], + [null, [null], null, null, null, null, null], + [6, [7], [8], [17, 18], [[14, 15]], 9, 10]])"; + std::shared_ptr data_type = make_type("UTC", arrow::TimeUnit::MICRO); + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON(data_type, json).ValueOrDie(); + ASSERT_EQ(NestedProjectionUtils::AlignTimestampTimezones(array, array->type()).get(), + array.get()); + + std::shared_ptr read_type = make_type("Asia/Shanghai", arrow::TimeUnit::MILLI); + std::shared_ptr aligned = + NestedProjectionUtils::AlignTimestampTimezones(array, read_type); + auto expected_type = arrow::struct_({ + arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai")), + arrow::field("row", + arrow::struct_({arrow::field( + "ltz", arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"))})), + arrow::field("list", + arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai"))), + arrow::field( + "fixed_list", + arrow::fixed_size_list(arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai"), 2)), + arrow::field("map", arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"), + arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai"))), + arrow::field("other_unit", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")), + arrow::field("ntz", arrow::timestamp(arrow::TimeUnit::MICRO)), + }); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, json).ValueOrDie(); + ASSERT_TRUE(aligned->type()->Equals(expected_type)) << aligned->type()->ToString(); + ASSERT_TRUE(aligned->Equals(*expected)); + ASSERT_TRUE(array->type()->Equals(data_type)) << array->type()->ToString(); + + std::shared_ptr aligned_slice = + NestedProjectionUtils::AlignTimestampTimezones(array->Slice(1), read_type); + ASSERT_TRUE(aligned_slice->Equals(*expected->Slice(1))); + + std::shared_ptr zoned = + arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + "[1, null]") + .ValueOrDie(); + ASSERT_EQ(NestedProjectionUtils::AlignTimestampTimezones( + zoned, arrow::timestamp(arrow::TimeUnit::MICRO)) + .get(), + zoned.get()); + std::shared_ptr unzoned = + arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::MICRO), + "[1, null]") + .ValueOrDie(); + ASSERT_EQ(NestedProjectionUtils::AlignTimestampTimezones( + unzoned, arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")) + .get(), + unzoned.get()); +} + +TEST(NestedProjectionUtilsTest, AlignTimestampTimezonesMatchesStructChildrenByName) { + auto utc = arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + auto shanghai = arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai"); + const std::string json = "[[1, 2], [null, null]]"; + std::shared_ptr array = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({arrow::field("unread", utc), arrow::field("read", utc)}), json) + .ValueOrDie(); + std::shared_ptr aligned = NestedProjectionUtils::AlignTimestampTimezones( + array, arrow::struct_({arrow::field("read", shanghai)})); + auto expected_type = + arrow::struct_({arrow::field("unread", utc), arrow::field("read", shanghai)}); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, json).ValueOrDie(); + ASSERT_TRUE(aligned->type()->Equals(expected_type)) << aligned->type()->ToString(); + ASSERT_TRUE(aligned->Equals(*expected)); +} + TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { auto file_schema = arrow::schema({ MakeField("f0", arrow::int32(), 1), diff --git a/src/paimon/format/lance/lance_file_batch_reader.cpp b/src/paimon/format/lance/lance_file_batch_reader.cpp index b1e0163ef..c2606afa8 100644 --- a/src/paimon/format/lance/lance_file_batch_reader.cpp +++ b/src/paimon/format/lance/lance_file_batch_reader.cpp @@ -128,9 +128,14 @@ Result LanceFileBatchReader::AlignBatch(ReadBatch batch) if (array->type_id() != arrow::Type::STRUCT) { return Status::Invalid("Lance reader returned a non-struct Arrow batch"); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr aligned, - NestedProjectionUtils::AlignArrayToReadType( - array, arrow::struct_(read_schema_->fields()), arrow_pool_.get())); + std::shared_ptr read_type = arrow::struct_(read_schema_->fields()); + // Zoned timestamps come back as UTC, as LanceFormatWriter and Paimon Java store them, while + // paimon asks for the local zone; the instants are the same. + std::shared_ptr relabelled = + NestedProjectionUtils::AlignTimestampTimezones(array, read_type); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(relabelled, read_type, arrow_pool_.get())); auto out_array = std::make_unique<::ArrowArray>(); auto out_schema = std::make_unique<::ArrowSchema>(); PAIMON_RETURN_NOT_OK_FROM_ARROW( diff --git a/src/paimon/format/lance/lance_file_format_test.cpp b/src/paimon/format/lance/lance_file_format_test.cpp index 0805d49e1..cdee8831d 100644 --- a/src/paimon/format/lance/lance_file_format_test.cpp +++ b/src/paimon/format/lance/lance_file_format_test.cpp @@ -182,6 +182,19 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { name, arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, arrow::key_value_metadata({DataType::kTimePrecision}, {std::to_string(precision)})); }; + auto ltz_fields = [](const std::string& zone) { + return arrow::FieldVector{ + arrow::field("ltz_0_col", arrow::timestamp(arrow::TimeUnit::SECOND, zone)), + arrow::field("ltz_3_col", arrow::timestamp(arrow::TimeUnit::MILLI, zone)), + arrow::field("ltz_6_col", arrow::timestamp(arrow::TimeUnit::MICRO, zone)), + arrow::field("ltz_9_col", arrow::timestamp(arrow::TimeUnit::NANO, zone)), + arrow::field("ltz_array_col", + arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, zone))), + arrow::field("ltz_row_col", + arrow::struct_({arrow::field( + "value", arrow::timestamp(arrow::TimeUnit::MILLI, zone))})), + }; + }; arrow::FieldVector fields = { arrow::field("bool_col", arrow::boolean()), arrow::field("i8_col", arrow::int8()), @@ -206,17 +219,21 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { arrow::field("time_row_array_col", arrow::struct_({arrow::field("values", arrow::list(time_field("item", 9)))})), }; + arrow::FieldVector local_ltz_fields = ltz_fields("GMT-10:00"); + fields.insert(fields.end(), local_ltz_fields.begin(), local_ltz_fields.end()); std::shared_ptr schema = arrow::schema(fields); std::shared_ptr expected = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ [true,-1,-2,-3,-4,1.5,2.5,"one","abc",1,0, "1970-01-01 00:00:00.000001","12.34",[1.0,2.0],[7],[3.0,4.0], - [0,null,45296000],[45296123],[[3723123,null]]], + [0,null,45296000],[45296123],[[3723123,null]], + 1700000000,1700000000123,1700000000123456,1700000000123456789, + [1700000000123456,null,-1],[1700000000123]], [false,1,2,3,4,3.5,4.5,"two","xyz",2,86399999, "2030-12-31 23:59:59.999999","-12.34",[],[8],[5.0,6.0], - [],[86399999],[[]]], + [],[86399999],[[]],-1,-1,-1,-1,[],[-1]], [null,null,null,null,null,null,null,null,null,null,null,null,null,null,[null],null, - null,[null],[null]] + null,[null],[null],null,null,null,null,null,[null]] ])") .ValueOrDie(); std::string path = PathUtil::JoinPath(directory_->Str(), "supported-types.lance"); @@ -246,6 +263,17 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { std::shared_ptr batch_array = std::move(batch_result).MoveValueUnsafe(); ASSERT_TRUE(batch_array->type()->Equals(arrow::struct_(fields), /*check_metadata=*/true)) << batch_array->type()->ToString(/*show_metadata=*/true); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> c_file_schema, reader->GetFileSchema()); + arrow::Result> file_schema = + arrow::ImportSchema(c_file_schema.get()); + ASSERT_TRUE(file_schema.ok()) << file_schema.status().ToString(); + for (const std::shared_ptr& utc_field : ltz_fields("UTC")) { + std::shared_ptr file_field = + file_schema.ValueUnsafe()->GetFieldByName(utc_field->name()); + ASSERT_NE(file_field, nullptr) << utc_field->name(); + ASSERT_TRUE(file_field->type()->Equals(utc_field->type())) << file_field->ToString(); + } } TEST_F(LanceFileFormatTest, RejectsNullTopLevelRows) { diff --git a/src/paimon/format/lance/lance_format_writer.cpp b/src/paimon/format/lance/lance_format_writer.cpp index 6fb0c1634..0e7776c41 100644 --- a/src/paimon/format/lance/lance_format_writer.cpp +++ b/src/paimon/format/lance/lance_format_writer.cpp @@ -26,6 +26,7 @@ #include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/type.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -64,12 +65,44 @@ Status ValidateRowValues(const std::shared_ptr& array) { return Status::OK(); } +// Lance 0.39 stores a zoned timestamp as `timestamp::` and cannot parse a zone id +// containing ':' (e.g. GMT-10:00), so store every zone as UTC, as Paimon Java does. The values +// are instants since the epoch whatever the zone, so only the type changes. +std::shared_ptr ToLanceType(const std::shared_ptr& type) { + switch (type->id()) { + case arrow::Type::TIMESTAMP: { + const auto& timestamp_type = checked_cast(*type); + if (timestamp_type.timezone().empty()) { + return type; + } + return arrow::timestamp(timestamp_type.unit(), "UTC"); + } + case arrow::Type::LIST: + return arrow::list(type->field(0)->WithType(ToLanceType(type->field(0)->type()))); + case arrow::Type::STRUCT: { + arrow::FieldVector fields; + fields.reserve(type->num_fields()); + for (const std::shared_ptr& field : type->fields()) { + fields.push_back(field->WithType(ToLanceType(field->type()))); + } + return arrow::struct_(fields); + } + default: + return type; + } +} + +std::shared_ptr ToLanceSchema(const std::shared_ptr& schema) { + return arrow::schema(ToLanceType(arrow::struct_(schema->fields()))->fields(), + schema->metadata()); +} + } // namespace LanceFormatWriter::LanceFormatWriter(const std::shared_ptr& schema, const std::shared_ptr& arrow_pool, PaimonLanceWriter* writer) - : schema_(schema), + : schema_(ToLanceSchema(schema)), arrow_pool_(arrow_pool), writer_(writer), metrics_(std::make_shared()) {} diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.cpp b/src/paimon/format/vortex/vortex_file_batch_reader.cpp index 6fa537b9b..e171589e5 100644 --- a/src/paimon/format/vortex/vortex_file_batch_reader.cpp +++ b/src/paimon/format/vortex/vortex_file_batch_reader.cpp @@ -168,68 +168,6 @@ Result> NormalizeViewArray(const std::shared_ptr AlignTimestampTimezones( - const std::shared_ptr& data, - const std::shared_ptr& read_type) { - const std::shared_ptr& type = data->type; - if (type->id() == arrow::Type::TIMESTAMP && read_type->id() == arrow::Type::TIMESTAMP) { - const auto& stored = checked_cast(*type); - const auto& wanted = checked_cast(*read_type); - if (stored.unit() != wanted.unit() || stored.timezone().empty() || - wanted.timezone().empty() || stored.timezone() == wanted.timezone()) { - return data; - } - std::shared_ptr retyped = data->Copy(); - retyped->type = arrow::timestamp(wanted.unit(), wanted.timezone()); - return retyped; - } - if (type->id() == arrow::Type::STRUCT && read_type->id() == arrow::Type::STRUCT) { - const auto& wanted_struct = checked_cast(*read_type); - arrow::FieldVector fields = type->fields(); - std::shared_ptr retyped; - for (int32_t i = 0; i < static_cast(data->child_data.size()); ++i) { - const int32_t index = wanted_struct.GetFieldIndex(fields[i]->name()); - if (index < 0) { - continue; - } - std::shared_ptr child = - AlignTimestampTimezones(data->child_data[i], wanted_struct.field(index)->type()); - if (child == data->child_data[i]) { - continue; - } - if (retyped == nullptr) { - retyped = data->Copy(); - } - retyped->child_data[i] = child; - fields[i] = fields[i]->WithType(child->type); - } - if (retyped == nullptr) { - return data; - } - retyped->type = arrow::struct_(fields); - return retyped; - } - if ((type->id() == arrow::Type::LIST || type->id() == arrow::Type::FIXED_SIZE_LIST) && - type->id() == read_type->id() && !data->child_data.empty()) { - std::shared_ptr values = - AlignTimestampTimezones(data->child_data[0], read_type->field(0)->type()); - if (values == data->child_data[0]) { - return data; - } - std::shared_ptr retyped = data->Copy(); - retyped->child_data[0] = values; - const std::shared_ptr value_field = type->field(0)->WithType(values->type); - retyped->type = type->id() == arrow::Type::LIST - ? arrow::list(value_field) - : arrow::fixed_size_list( - value_field, - checked_cast(*type).list_size()); - return retyped; - } - return data; -} - } // namespace VortexFileBatchReader::VortexFileBatchReader( @@ -395,8 +333,9 @@ Result> VortexFileBatchReader::AlignToReadSchema( } // TODO(zhouyc-ali): push top-level column pruning into the Vortex scan std::shared_ptr read_type = arrow::struct_(read_schema_->fields()); + // Vortex reports the file's zone and paimon asks for the reader's; the instants are the same. return NestedProjectionUtils::AlignArrayToReadType( - arrow::MakeArray(AlignTimestampTimezones(array->data(), read_type)), read_type, + NestedProjectionUtils::AlignTimestampTimezones(array, read_type), read_type, arrow_pool_.get()); } diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 3bfb58c7e..17807e254 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -3637,6 +3637,73 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { check(1, {"f_struct", "id", "f_vector", "f_array_int"}, projected->Slice(0, 2)); check(2, {"f_struct", "id", "f_vector", "f_array_int"}, projected); } + +TEST_F(ScanAndReadInteTest, TestLanceLtzCompatibility) { + TimezoneGuard timezone_guard("Asia/Shanghai"); + const std::string zone = DateTimeUtils::GetLocalTimezoneName(); + arrow::FieldVector fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("f_ltz_0", arrow::timestamp(arrow::TimeUnit::SECOND, zone)), + arrow::field("f_ltz_3", arrow::timestamp(arrow::TimeUnit::MILLI, zone)), + arrow::field("f_ltz_6", arrow::timestamp(arrow::TimeUnit::MICRO, zone)), + arrow::field("f_ltz_9", arrow::timestamp(arrow::TimeUnit::NANO, zone)), + arrow::field("f_ltz_array", arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, zone))), + arrow::field( + "f_ltz_struct", + arrow::struct_({arrow::field("number", arrow::int32()), + arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MILLI, zone))})), + }; + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +[0, 1, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", ["1970-01-01 00:00:01.123456", null, "1969-12-31 23:59:59.999999"], [1, "1970-01-01 00:00:01.123"]], +[0, 2, null, null, null, null, null, [null, null]], +[0, 10, "1969-12-31 00:00:00", "1969-12-31 23:59:59.999", "1969-12-31 23:59:59.999999", "1969-12-31 23:59:59.999999999", [], [10, "1969-12-31 23:59:59.999"]], +[0, 11, "2024-10-04 12:34:56", "2024-10-04 12:34:56.123", "2024-10-04 12:34:56.123456", "2024-10-04 12:34:56.123456789", ["2024-10-04 12:34:56.123456"], [11, "2024-10-04 12:34:56.123"]] +])") + .ValueOrDie()); + auto projected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[7], fields[1], fields[5], fields[6]}), R"([ +[0, [1, "1970-01-01 00:00:01.123"], 1, "1970-01-01 00:00:01.123456789", ["1970-01-01 00:00:01.123456", null, "1969-12-31 23:59:59.999999"]], +[0, [null, null], 2, null, null], +[0, [10, "1969-12-31 23:59:59.999"], 10, "1969-12-31 23:59:59.999999999", []], +[0, [11, "2024-10-04 12:34:56.123"], 11, "2024-10-04 12:34:56.123456789", ["2024-10-04 12:34:56.123456"]] +])") + .ValueOrDie()); + auto check = [&](const std::string& table_path, int64_t snapshot_id, + const std::vector& projection, + const std::shared_ptr& expected_result) { + SCOPED_TRACE("snapshot " + std::to_string(snapshot_id)); + ScanContextBuilder scan_builder(table_path); + scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + ASSERT_EQ(plan->SnapshotId(), std::optional(snapshot_id)); + ASSERT_FALSE(plan->Splits().empty()); + + ReadContextBuilder read_builder(table_path); + read_builder.AddOption("read.batch-size", "1"); + if (!projection.empty()) { + read_builder.SetReadFieldNames(projection); + } + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); + ASSERT_TRUE(actual); + ASSERT_TRUE(expected_result->Equals(actual)) + << "actual: " << actual->ToString() << "\nexpected: " << expected_result->ToString(); + }; + for (const std::string table_name : {"append_ltz_java_compat", "append_ltz_python_compat"}) { + SCOPED_TRACE(table_name); + std::string table_path = GetDataDir() + "/lance/" + table_name + ".db/" + table_name; + check(table_path, 1, /*projection=*/{}, expected->Slice(0, 2)); + check(table_path, 2, /*projection=*/{}, expected); + check(table_path, 2, {"f_ltz_struct", "id", "f_ltz_9", "f_ltz_array"}, projected); + } +} #endif #ifdef PAIMON_ENABLE_MOSAIC diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 8b36d3c72..dbbdf63f6 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -2589,7 +2590,7 @@ TEST_P(WriteAndReadInteTest, TestAppendTimestampType) { }; auto schema = arrow::schema(fields); auto [file_format, file_system] = GetParam(); - if (file_format == "mosaic" || file_format == "lance") { + if (file_format == "mosaic") { return; } std::map options = {{Options::FILE_FORMAT, file_format}, @@ -2646,7 +2647,7 @@ TEST_P(WriteAndReadInteTest, TestPkTimestampType) { }; auto schema = arrow::schema(fields); auto [file_format, file_system] = GetParam(); - if (file_format == "mosaic" || file_format == "lance") { + if (file_format == "mosaic") { return; } std::map options = {{Options::FILE_FORMAT, file_format}, @@ -6027,6 +6028,139 @@ TEST(TimeColumnWriteInteTest, TestAppendTimeColumnFinalize) { } #endif +#ifdef PAIMON_ENABLE_LANCE +class LanceTimestampLtzInteTest + : public ::testing::Test, + public ::testing::WithParamInterface> {}; + +TEST_P(LanceTimestampLtzInteTest, TestWriteAndReadInPinnedTimezone) { + TimezoneGuard timezone_guard(std::get<0>(GetParam())); + const std::string timezone = DateTimeUtils::GetLocalTimezoneName(); + const bool with_primary_key = std::get<1>(GetParam()); + auto dir = UniqueTestDirectory::Create("local"); + ASSERT_NE(dir, nullptr); + + auto ltz = [&timezone](arrow::TimeUnit::type unit) { return arrow::timestamp(unit, timezone); }; + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("ltz_0", ltz(arrow::TimeUnit::SECOND)), + arrow::field("ltz_9", ltz(arrow::TimeUnit::NANO)), + arrow::field("ltz_array", arrow::list(ltz(arrow::TimeUnit::MICRO))), + arrow::field( + "ltz_row", + arrow::struct_({arrow::field("ltz_3", ltz(arrow::TimeUnit::MILLI)), + arrow::field("ntz", arrow::timestamp(arrow::TimeUnit::MICRO))})), + arrow::field("ntz_to_ltz", arrow::timestamp(arrow::TimeUnit::MICRO)), + arrow::field("ltz_to_ntz", ltz(arrow::TimeUnit::MICRO)), + }; + std::map options = {{Options::FILE_FORMAT, "lance"}, + {Options::BUCKET, with_primary_key ? "1" : "-1"}, + {Options::FILE_SYSTEM, "local"}}; + std::vector primary_keys; + if (with_primary_key) { + primary_keys.push_back("id"); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), arrow::schema(fields), + /*partition_keys=*/{}, primary_keys, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([ + [1, "1969-12-31 23:59:59", "1969-12-31 23:59:59.999999999", + ["2024-10-04 12:34:56.123456", null], + ["1970-01-01 00:00:00.001", "1970-01-01 00:00:00.000001"], + "1969-12-31 23:59:59.999999", "1970-01-01 00:00:00.000001"], + [2, null, null, null, [null, null], null, null], + [3, "2024-10-04 12:34:56", "2024-10-04 12:34:56.123456789", [], + ["2024-10-04 12:34:56.123", null], "2024-10-04 12:34:56.123456", + "2024-10-04 12:34:56.123456"] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + SchemaManager schema_manager(dir->GetFileSystem(), table_path); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema_v0, schema_manager.ReadSchema(0)); + const int32_t added_field_id = schema_v0->HighestFieldId() + 1; + std::shared_ptr added_field = + arrow::field("ltz_added", ltz(arrow::TimeUnit::MICRO)); + arrow::FieldVector fields_v1 = fields; + fields_v1[5] = fields[5]->WithType(ltz(arrow::TimeUnit::MICRO)); + fields_v1[6] = fields[6]->WithType(arrow::timestamp(arrow::TimeUnit::MICRO)); + fields_v1.push_back(added_field); + std::vector data_fields_v1 = schema_v0->Fields(); + data_fields_v1[5] = DataField(data_fields_v1[5].Id(), fields_v1[5]); + data_fields_v1[6] = DataField(data_fields_v1[6].Id(), fields_v1[6]); + data_fields_v1.emplace_back(added_field_id, added_field); + ASSERT_OK(TestHelper::WriteNextSchema(dir->GetFileSystem(), table_path, data_fields_v1, + added_field_id, options)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(table_path, options, /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v1), R"([ + [4, "1970-01-01 00:00:00", null, [null], [null, "1969-12-31 23:59:59.999999"], + "1969-12-31 23:59:59.999999", "1970-01-01 00:00:00.000001", + "2024-10-04 12:34:56.123456"] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadFieldNames( + {"ltz_row", "ltz_added", "ltz_9", "id", "ltz_array", "ltz_0", "ltz_to_ntz", "ntz_to_ltz"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(splits)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(std::move(batch_reader))); + const std::map> converted_values = { + {"UTC", + {"1970-01-01 00:00:00.000001", "1969-12-31 23:59:59.999999", "2024-10-04 12:34:56.123456", + "2024-10-04 12:34:56.123456"}}, + {"Asia/Shanghai", + {"1970-01-01 08:00:00.000001", "1969-12-31 15:59:59.999999", "2024-10-04 20:34:56.123456", + "2024-10-04 04:34:56.123456"}}, + {"Pacific/Honolulu", + {"1969-12-31 14:00:00.000001", "1970-01-01 09:59:59.999999", "2024-10-04 02:34:56.123456", + "2024-10-04 22:34:56.123456"}}, + }; + const std::vector& converted = converted_values.at(std::get<0>(GetParam())); + auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), fields_v1[4], + fields_v1[7], fields_v1[2], fields_v1[0], fields_v1[3], + fields_v1[1], fields_v1[6], fields_v1[5]}); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON( + expected_type, fmt::format(R"([ + [0, ["1970-01-01 00:00:00.001", "1970-01-01 00:00:00.000001"], null, + "1969-12-31 23:59:59.999999999", 1, ["2024-10-04 12:34:56.123456", null], + "1969-12-31 23:59:59", "{}", "{}"], + [0, [null, null], null, null, 2, null, null, null, null], + [0, ["2024-10-04 12:34:56.123", null], null, "2024-10-04 12:34:56.123456789", 3, [], + "2024-10-04 12:34:56", "{}", "{}"], + [0, [null, "1969-12-31 23:59:59.999999"], "2024-10-04 12:34:56.123456", null, 4, + [null], "1970-01-01 00:00:00", "1970-01-01 00:00:00.000001", + "1969-12-31 23:59:59.999999"] + ])", + converted[0], converted[1], converted[2], converted[3])) + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(actual)) + << "actual: " << actual->ToString() << "\nexpected: " << expected->ToString(); +} + +INSTANTIATE_TEST_SUITE_P(TimezoneAndPrimaryKey, LanceTimestampLtzInteTest, + ::testing::Combine(::testing::Values(std::string("UTC"), + std::string("Asia/Shanghai"), + std::string("Pacific/Honolulu")), + ::testing::Bool())); +#endif + INSTANTIATE_TEST_SUITE_P(FileFormatAndFileSystem, WriteAndReadInteTest, ::testing::ValuesIn(GetTestValuesForWriteAndReadInteTest())); diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/README.md b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/README.md new file mode 100644 index 000000000..2e808e92a --- /dev/null +++ b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/README.md @@ -0,0 +1,40 @@ +Table: append_ltz_java_compat +Writer: Paimon Java 2.2-SNAPSHOT at commit 535a5c944f854cb8b68196d44677e576cf22120b +Lance version: lance-core 0.39.0 +Arrow Java version: 15.0.0 +JVM default time zone: GMT-10:00 + +Schema: +id INT +f_ltz_0 TIMESTAMP(0) WITH LOCAL TIME ZONE +f_ltz_3 TIMESTAMP(3) WITH LOCAL TIME ZONE +f_ltz_6 TIMESTAMP(6) WITH LOCAL TIME ZONE +f_ltz_9 TIMESTAMP(9) WITH LOCAL TIME ZONE +f_ltz_array ARRAY +f_ltz_struct ROW + +Options: +bucket = -1 +file.format = lance +metadata.stats-mode = none + +This unpartitioned append-only table has no primary key. The Java catalog and +BatchTableWrite/BatchTableCommit APIs wrote two rows per commit into two Lance data files, +with schema, Avro manifests and two snapshots. All four rows were read back and verified in Java. +Values below follow schema order; timestamps are UTC. + +Add: (1, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, [1970-01-01 00:00:01.123456, NULL, 1969-12-31 23:59:59.999999], (1, 1970-01-01 00:00:01.123)) +Add: (2, NULL, NULL, NULL, NULL, NULL, (NULL, NULL)) + +Commit - snapshot 1 (2 rows) + +Add: (10, 1969-12-31 00:00:00, 1969-12-31 23:59:59.999, 1969-12-31 23:59:59.999999, 1969-12-31 23:59:59.999999999, [], (10, 1969-12-31 23:59:59.999)) +Add: (11, 2024-10-04 12:34:56, 2024-10-04 12:34:56.123, 2024-10-04 12:34:56.123456, 2024-10-04 12:34:56.123456789, [2024-10-04 12:34:56.123456], (11, 2024-10-04 12:34:56.123)) + +Commit - snapshot 2 (4 rows total) + +The writer ran with a default time zone containing ':', which Lance 0.39 cannot parse as a +time zone id. Java writes every local-zoned timestamp, including the ARRAY element and the +ROW child, to Lance as a timestamp with the "UTC" time zone, leaving the instants unchanged. +ROW parents are always valid, including id=2 where both children are NULL, because Lance 0.39 +does not preserve null ROW parents. diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-4fcb4a78-f17b-4a9c-9454-ea37ee78e0c5-0.lance b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-4fcb4a78-f17b-4a9c-9454-ea37ee78e0c5-0.lance new file mode 100644 index 000000000..d4b8fd8bb Binary files /dev/null and b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-4fcb4a78-f17b-4a9c-9454-ea37ee78e0c5-0.lance differ diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-e9ac3746-5001-45e0-8060-d0e84efae172-0.lance b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-e9ac3746-5001-45e0-8060-d0e84efae172-0.lance new file mode 100644 index 000000000..b6bdcade6 Binary files /dev/null and b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-e9ac3746-5001-45e0-8060-d0e84efae172-0.lance differ diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-1e2d27b3-41db-46d5-9c0e-507c8ffe695a-0 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-1e2d27b3-41db-46d5-9c0e-507c8ffe695a-0 new file mode 100644 index 000000000..1c8b8ffcf Binary files /dev/null and b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-1e2d27b3-41db-46d5-9c0e-507c8ffe695a-0 differ diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-fe3d8af9-46b4-4157-b164-7a0660316cf5-0 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-fe3d8af9-46b4-4157-b164-7a0660316cf5-0 new file mode 100644 index 000000000..9fb3d4ed1 Binary files /dev/null and b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-fe3d8af9-46b4-4157-b164-7a0660316cf5-0 differ diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0 new file mode 100644 index 000000000..c14fca8c8 Binary files /dev/null and b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0 differ diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1 new file mode 100644 index 000000000..a86721c24 Binary files /dev/null and b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1 differ diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0 new file mode 100644 index 000000000..555dd609f Binary files /dev/null and b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0 differ diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1 new file mode 100644 index 000000000..4fbf2b487 Binary files /dev/null and b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1 differ diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/schema/schema-0 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/schema/schema-0 new file mode 100644 index 000000000..af124f64a --- /dev/null +++ b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/schema/schema-0 @@ -0,0 +1,56 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "f_ltz_0", + "type" : "TIMESTAMP(0) WITH LOCAL TIME ZONE" + }, { + "id" : 2, + "name" : "f_ltz_3", + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + }, { + "id" : 3, + "name" : "f_ltz_6", + "type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE" + }, { + "id" : 4, + "name" : "f_ltz_9", + "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE" + }, { + "id" : 5, + "name" : "f_ltz_array", + "type" : { + "type" : "ARRAY", + "element" : "TIMESTAMP(6) WITH LOCAL TIME ZONE" + } + }, { + "id" : 6, + "name" : "f_ltz_struct", + "type" : { + "type" : "ROW", + "fields" : [ { + "id" : 7, + "name" : "number", + "type" : "INT" + }, { + "id" : 8, + "name" : "ts", + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + } ] + } + } ], + "highestFieldId" : 8, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "file.format" : "lance", + "metadata.stats-mode" : "none" + }, + "timeMillis" : 1793201358194 +} \ No newline at end of file diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/EARLIEST b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/LATEST b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/LATEST new file mode 100644 index 000000000..d8263ee98 --- /dev/null +++ b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/LATEST @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-1 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-1 new file mode 100644 index 000000000..e93e18174 --- /dev/null +++ b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "cd56a571-077d-420f-ac22-695717c200de", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1", + "deltaManifestListSize" : 1267, + "commitUser" : "41aabf22-c463-441a-9c3a-bfc9f6852590", + "writerVersion" : "java-2.2-SNAPSHOT-535a5c944f854cb8b68196d44677e576cf22120b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1793201360419, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-2 b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-2 new file mode 100644 index 000000000..ecfa750e4 --- /dev/null +++ b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "10e95ecd-534f-4016-9644-e37ce4fb51ec", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0", + "baseManifestListSize" : 1267, + "deltaManifestList" : "manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1", + "deltaManifestListSize" : 1267, + "commitUser" : "44116caa-dcc7-4397-b617-fa43046b55b8", + "writerVersion" : "java-2.2-SNAPSHOT-535a5c944f854cb8b68196d44677e576cf22120b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1793201360592, + "totalRecordCount" : 4, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/README.md b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/README.md new file mode 100644 index 000000000..f48c85119 --- /dev/null +++ b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/README.md @@ -0,0 +1,43 @@ +Table: append_ltz_python_compat +Writer: Apache Paimon Python at commit 63ffd403648e1cf9848e32bb29a398c1b3b16ab1 +Source: https://github.com/apache/paimon/tree/63ffd403648e1cf9848e32bb29a398c1b3b16ab1/paimon-python +Python version: 3.11.15 +Lance version: pylance 0.39.0 +Arrow Python version: 19.0.1 + +This unpartitioned append-only table has no primary key. The Python Catalog and batch +table-write/commit APIs wrote two rows per commit into two Lance files, with Avro manifests +and two snapshots. All values were read back in Python after each commit; reordered +projection was also verified after snapshot 2. + +Schema (all fields nullable): +id INT +f_ltz_0 TIMESTAMP_LTZ(0) +f_ltz_3 TIMESTAMP_LTZ(3) +f_ltz_6 TIMESTAMP_LTZ(6) +f_ltz_9 TIMESTAMP_LTZ(9) +f_ltz_array ARRAY +f_ltz_struct ROW + +Options: +bucket = -1 +file.format = lance +manifest.format = avro +metadata.stats-mode = none + +Values below follow schema order; timestamps are UTC. They match append_ltz_java_compat. + +Add: (1, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, [1970-01-01 00:00:01.123456, NULL, 1969-12-31 23:59:59.999999], (1, 1970-01-01 00:00:01.123)) +Add: (2, NULL, NULL, NULL, NULL, NULL, (NULL, NULL)) + +Commit - snapshot 1 (2 rows) + +Add: (10, 1969-12-31 00:00:00, 1969-12-31 23:59:59.999, 1969-12-31 23:59:59.999999, 1969-12-31 23:59:59.999999999, [], (10, 1969-12-31 23:59:59.999)) +Add: (11, 2024-10-04 12:34:56, 2024-10-04 12:34:56.123, 2024-10-04 12:34:56.123456, 2024-10-04 12:34:56.123456789, [2024-10-04 12:34:56.123456], (11, 2024-10-04 12:34:56.123)) + +Commit - snapshot 2 (4 rows total) + +Paimon Python maps a pyarrow timestamp with a time zone to TIMESTAMP_LTZ and writes every +TIMESTAMP_LTZ, including the ARRAY element and the ROW child, to Lance as a timestamp with +the "UTC" time zone. +ROW parents are always valid: this fixture does not claim null-parent ROW compatibility. diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-3dd49f37-400b-4d87-ab60-dda8278083c8-0.lance b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-3dd49f37-400b-4d87-ab60-dda8278083c8-0.lance new file mode 100644 index 000000000..a88ebd178 Binary files /dev/null and b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-3dd49f37-400b-4d87-ab60-dda8278083c8-0.lance differ diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-cb6cba6a-5d96-42f4-b8b9-2b5b4616c60f-0.lance b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-cb6cba6a-5d96-42f4-b8b9-2b5b4616c60f-0.lance new file mode 100644 index 000000000..0fbf94bb8 Binary files /dev/null and b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-cb6cba6a-5d96-42f4-b8b9-2b5b4616c60f-0.lance differ diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-2630e922-a399-4613-9181-54cc65803426-0 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-2630e922-a399-4613-9181-54cc65803426-0 new file mode 100644 index 000000000..583437a5e Binary files /dev/null and b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-2630e922-a399-4613-9181-54cc65803426-0 differ diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-db163726-5626-4b91-a607-33f79a420829-0 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-db163726-5626-4b91-a607-33f79a420829-0 new file mode 100644 index 000000000..54f23577e Binary files /dev/null and b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-db163726-5626-4b91-a607-33f79a420829-0 differ diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-0 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-0 new file mode 100644 index 000000000..03a8e842a Binary files /dev/null and b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-0 differ diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-1 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-1 new file mode 100644 index 000000000..78e113ffd Binary files /dev/null and b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-1 differ diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0 new file mode 100644 index 000000000..d7aaffb2b Binary files /dev/null and b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0 differ diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1 new file mode 100644 index 000000000..0f034d16b Binary files /dev/null and b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1 differ diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/schema/schema-0 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/schema/schema-0 new file mode 100644 index 000000000..ddf0dc4f1 --- /dev/null +++ b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/schema/schema-0 @@ -0,0 +1,71 @@ +{ + "version": 3, + "id": 0, + "fields": [ + { + "id": 0, + "name": "id", + "type": "INT" + }, + { + "id": 1, + "name": "f_ltz_0", + "type": "TIMESTAMP_LTZ(0)" + }, + { + "id": 2, + "name": "f_ltz_3", + "type": "TIMESTAMP_LTZ(3)" + }, + { + "id": 3, + "name": "f_ltz_6", + "type": "TIMESTAMP_LTZ(6)" + }, + { + "id": 4, + "name": "f_ltz_9", + "type": "TIMESTAMP_LTZ(9)" + }, + { + "id": 5, + "name": "f_ltz_array", + "type": { + "type": "ARRAY", + "element": "TIMESTAMP_LTZ(6)", + "nullable": true + } + }, + { + "id": 6, + "name": "f_ltz_struct", + "type": { + "type": "ROW", + "fields": [ + { + "id": 7, + "name": "number", + "type": "INT" + }, + { + "id": 8, + "name": "ts", + "type": "TIMESTAMP_LTZ(3)" + } + ], + "nullable": true + } + } + ], + "highestFieldId": 8, + "partitionKeys": [], + "primaryKeys": [], + "options": { + "bucket": "-1", + "file.format": "lance", + "manifest.format": "avro", + "metadata.stats-mode": "none" + }, + "comment": null, + "timeMillis": 1793201390242 +} \ No newline at end of file diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/LATEST b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/LATEST new file mode 100644 index 000000000..d8263ee98 --- /dev/null +++ b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/LATEST @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-1 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-1 new file mode 100644 index 000000000..337aa98c0 --- /dev/null +++ b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-1 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 1, + "schemaId": 0, + "baseManifestList": "manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0", + "deltaManifestList": "manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1", + "totalRecordCount": 2, + "deltaRecordCount": 2, + "commitUser": "bf1553ec-84d3-4bba-a63f-af57a0821aac", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1793201390271, + "uuid": "146996fb-0a39-443e-8c5b-8716ef072816", + "writerVersion": "python-2.1.dev-63ffd403648e1cf9848e32bb29a398c1b3b16ab1" +} \ No newline at end of file diff --git a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-2 b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-2 new file mode 100644 index 000000000..450f7d730 --- /dev/null +++ b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-2 @@ -0,0 +1,15 @@ +{ + "version": 3, + "id": 2, + "schemaId": 0, + "baseManifestList": "manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-0", + "deltaManifestList": "manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-1", + "totalRecordCount": 4, + "deltaRecordCount": 2, + "commitUser": "a1338729-f60a-4d94-895b-8c4860517802", + "commitIdentifier": 9223372036854775807, + "commitKind": "APPEND", + "timeMillis": 1793201390281, + "uuid": "d721aff9-8b3f-4e4a-82ad-b522916f5e08", + "writerVersion": "python-2.1.dev-63ffd403648e1cf9848e32bb29a398c1b3b16ab1" +} \ No newline at end of file