From 123f0f8c283a4a3b250742eebf7d88a1b5604b93 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Tue, 27 Oct 2026 20:55:55 -0700 Subject: [PATCH] feat(lance): support TIME type Paimon Java accepts TIME columns in Lance data files and writes them as time32[ms], but Paimon C++ rejected TIME in Lance schema validation, both at top level and nested in ARRAY or ROW. Accept TIME there, and create empty integer statistics for a TIME column in LanceStatsExtractor so that finalizing a Lance data file with one does not fail. Document TIME in the data types guide, which still listed it as not supported. Describe its time32[ms] mapping, how the declared precision is kept and which file formats read or write it, and state that only append-only tables support it and that it cannot be a partition or bucket key or be referenced by predicates. --- docs/source/user_guide/data_types.rst | 18 ++++- src/paimon/core/schema/schema_validation.cpp | 1 + .../core/schema/schema_validation_test.cpp | 12 +-- .../format/lance/lance_file_format_test.cpp | 46 ++++++++++-- .../format/lance/lance_stats_extractor.cpp | 2 + test/inte/scan_and_read_inte_test.cpp | 24 +++--- test/inte/write_and_read_inte_test.cpp | 71 +++++++++++------- .../append_java_compat/README.md | 18 +++-- ...3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance} | Bin 8812 -> 9330 bytes ...161b4-2446-450b-9b59-2e880ced6aaa-0.lance} | Bin 10275 -> 10947 bytes ...st-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0} | Bin 2166 -> 2165 bytes ...st-428dd1a6-0756-44d3-939c-f39159964fbc-0} | Bin 2164 -> 2164 bytes ...st-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0} | Bin 1110 -> 1110 bytes ...st-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1} | Bin 1113 -> 1113 bytes ...st-24709406-a874-4961-b200-879ceba4452c-0} | Bin 1006 -> 1006 bytes ...st-24709406-a874-4961-b200-879ceba4452c-1} | Bin 1110 -> 1110 bytes .../append_java_compat/schema/schema-0 | 44 ++++++----- .../append_java_compat/snapshot/snapshot-1 | 10 +-- .../append_java_compat/snapshot/snapshot-2 | 10 +-- .../append_python_compat/README.md | 14 +++- ...c7753-16ed-451d-8899-984adb2de133-0.lance} | Bin 6717 -> 7187 bytes ...4eee6-062d-4781-b762-d18f2f1a90c8-0.lance} | Bin 8103 -> 8727 bytes ...st-63284fb6-53d2-4c5d-8719-a411c920e155-0} | Bin 2208 -> 2209 bytes ...st-a10d7552-9602-4543-bc69-152765c1c59c-0} | Bin 2208 -> 2208 bytes ...st-cb00c974-4152-4c3c-aa64-7910d413b4d6-0} | Bin 799 -> 799 bytes ...st-cb00c974-4152-4c3c-aa64-7910d413b4d6-1} | Bin 897 -> 897 bytes ...st-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0} | Bin 897 -> 897 bytes ...st-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1} | Bin 897 -> 897 bytes .../append_python_compat/schema/schema-0 | 40 ++++++---- .../append_python_compat/snapshot/snapshot-1 | 10 +-- .../append_python_compat/snapshot/snapshot-2 | 10 +-- 31 files changed, 213 insertions(+), 117 deletions(-) rename test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/{data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance => data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance} (75%) rename test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/{data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance => data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance} (78%) rename test/test_data/lance/append_java_compat.db/append_java_compat/manifest/{manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0 => manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0} (91%) rename test/test_data/lance/append_java_compat.db/append_java_compat/manifest/{manifest-ce7a697c-635e-420f-8014-af0166fa5eea-0 => manifest-428dd1a6-0756-44d3-939c-f39159964fbc-0} (91%) rename test/test_data/lance/append_java_compat.db/append_java_compat/manifest/{manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0 => manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0} (86%) rename test/test_data/lance/append_java_compat.db/append_java_compat/manifest/{manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1 => manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1} (86%) rename test/test_data/lance/append_java_compat.db/append_java_compat/manifest/{manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0 => manifest-list-24709406-a874-4961-b200-879ceba4452c-0} (95%) rename test/test_data/lance/append_java_compat.db/append_java_compat/manifest/{manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1 => manifest-list-24709406-a874-4961-b200-879ceba4452c-1} (86%) rename test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/{data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance => data-24cc7753-16ed-451d-8899-984adb2de133-0.lance} (71%) rename test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/{data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance => data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance} (75%) rename test/test_data/lance/append_python_compat.db/append_python_compat/manifest/{manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0 => manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0} (92%) rename test/test_data/lance/append_python_compat.db/append_python_compat/manifest/{manifest-6b2e90c5-eabd-41e9-9a02-099ecdb7fede-0 => manifest-a10d7552-9602-4543-bc69-152765c1c59c-0} (92%) rename test/test_data/lance/append_python_compat.db/append_python_compat/manifest/{manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0 => manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0} (96%) rename test/test_data/lance/append_python_compat.db/append_python_compat/manifest/{manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1 => manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1} (85%) rename test/test_data/lance/append_python_compat.db/append_python_compat/manifest/{manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0 => manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0} (85%) rename test/test_data/lance/append_python_compat.db/append_python_compat/manifest/{manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1 => manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1} (85%) diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index 34818ea49..92b1dda11 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -146,13 +146,29 @@ and `Arrow DataTypes expected_errors = {"type MAP", "LOCAL_ZONED_TIMESTAMP", "type VARIANT", - "type time32", "type time32"}; + std::vector expected_errors = {"type MAP", "LOCAL_ZONED_TIMESTAMP", + "type VARIANT"}; for (size_t i = 0; i < unsupported_fields.size(); ++i) { ASSERT_OK_AND_ASSIGN( table_schema, diff --git a/src/paimon/format/lance/lance_file_format_test.cpp b/src/paimon/format/lance/lance_file_format_test.cpp index 089f5f328..0805d49e1 100644 --- a/src/paimon/format/lance/lance_file_format_test.cpp +++ b/src/paimon/format/lance/lance_file_format_test.cpp @@ -29,6 +29,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" @@ -176,6 +177,11 @@ TEST(LanceUtilsTest, ConvertsObjectStoreOptions) { } TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { + auto time_field = [](const std::string& name, int32_t precision) { + return arrow::field( + name, arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, + arrow::key_value_metadata({DataType::kTimePrecision}, {std::to_string(precision)})); + }; arrow::FieldVector fields = { arrow::field("bool_col", arrow::boolean()), arrow::field("i8_col", arrow::int8()), @@ -187,6 +193,7 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { arrow::field("string_col", arrow::utf8()), arrow::field("binary_col", arrow::binary()), arrow::field("date_col", arrow::date32()), + time_field("time_col", 3), arrow::field("timestamp_col", arrow::timestamp(arrow::TimeUnit::MICRO)), arrow::field("decimal_col", arrow::decimal128(12, 2)), arrow::field("array_col", arrow::list(arrow::float32())), @@ -194,15 +201,22 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { arrow::field( "vector_col", arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 2)), + arrow::field("time_array_col", arrow::list(time_field("item", 0))), + arrow::field("time_row_col", arrow::struct_({time_field("value", 6)})), + arrow::field("time_row_array_col", + arrow::struct_({arrow::field("values", arrow::list(time_field("item", 9)))})), }; std::shared_ptr schema = arrow::schema(fields); std::shared_ptr expected = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ - [true,-1,-2,-3,-4,1.5,2.5,"one","abc",1, - "1970-01-01 00:00:00.000001","12.34",[1.0,2.0],[7],[3.0,4.0]], - [false,1,2,3,4,3.5,4.5,"two","xyz",2, - "2030-12-31 23:59:59.999999","-12.34",[],[8],[5.0,6.0]], - [null,null,null,null,null,null,null,null,null,null,null,null,null,[null],null] + [true,-1,-2,-3,-4,1.5,2.5,"one","abc",1,0, + "1970-01-01 00:00:00.000001","12.34",[1.0,2.0],[7],[3.0,4.0], + [0,null,45296000],[45296123],[[3723123,null]]], + [false,1,2,3,4,3.5,4.5,"two","xyz",2,86399999, + "2030-12-31 23:59:59.999999","-12.34",[],[8],[5.0,6.0], + [],[86399999],[[]]], + [null,null,null,null,null,null,null,null,null,null,null,null,null,null,[null],null, + null,[null],[null]] ])") .ValueOrDie(); std::string path = PathUtil::JoinPath(directory_->Str(), "supported-types.lance"); @@ -219,6 +233,19 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { ASSERT_TRUE(actual->Equals(arrow::ChunkedArray(expected))) << actual->ToString() << "\nvs\n" << expected->ToString(); } + + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, OpenReader(path, 2)); + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&ffi_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + arrow::Result> batch_result = + arrow::ImportArray(batch.first.get(), batch.second.get()); + ASSERT_TRUE(batch_result.ok()) << batch_result.status().ToString(); + std::shared_ptr batch_array = std::move(batch_result).MoveValueUnsafe(); + ASSERT_TRUE(batch_array->type()->Equals(arrow::struct_(fields), /*check_metadata=*/true)) + << batch_array->type()->ToString(/*show_metadata=*/true); } TEST_F(LanceFileFormatTest, RejectsNullTopLevelRows) { @@ -351,10 +378,12 @@ TEST_F(LanceFileFormatTest, EmptyProjectionAndEmptySelection) { TEST_F(LanceFileFormatTest, ExtractStatistics) { arrow::FieldVector fields = {arrow::field("id", arrow::int32()), - arrow::field("name", arrow::utf8())}; + arrow::field("name", arrow::utf8()), + arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI))}; std::shared_ptr schema = arrow::schema(fields); std::shared_ptr data = - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([[1,"a"],[2,null]])") + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([[1,"a",3723123],[2,null,null]])") .ValueOrDie(); std::string path = PathUtil::JoinPath(directory_->Str(), "stats.lance"); ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2)); @@ -365,9 +394,10 @@ TEST_F(LanceFileFormatTest, ExtractStatistics) { format_->CreateStatsExtractor(&ffi_schema)); ASSERT_OK_AND_ASSIGN(auto result, extractor->ExtractWithFileInfo(file_system_, path, pool_)); ASSERT_EQ(result.second.GetRowCount(), 2); - ASSERT_EQ(result.first.size(), 2); + ASSERT_EQ(result.first.size(), 3); ASSERT_EQ(result.first[0]->ToString(), "min null, max null, null count null"); ASSERT_EQ(result.first[1]->ToString(), "min null, max null, null count null"); + ASSERT_EQ(result.first[2]->ToString(), "min null, max null, null count null"); } TEST_F(LanceFileFormatTest, RejectsIncompatibleReadTypes) { diff --git a/src/paimon/format/lance/lance_stats_extractor.cpp b/src/paimon/format/lance/lance_stats_extractor.cpp index 0bd39f298..833af6a59 100644 --- a/src/paimon/format/lance/lance_stats_extractor.cpp +++ b/src/paimon/format/lance/lance_stats_extractor.cpp @@ -86,6 +86,8 @@ Result> LanceStatsExtractor::CreateEmptyStats( return ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::DATE32: return ColumnStats::CreateDateColumnStats(std::nullopt, std::nullopt, std::nullopt); + case arrow::Type::TIME32: + return ColumnStats::CreateIntColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::TIMESTAMP: { std::shared_ptr timestamp_type = checked_pointer_cast(type); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index b7cc73b96..3bfb58c7e 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -3479,6 +3479,8 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { arrow::field("f_binary", arrow::binary()), arrow::field("f_varbinary", arrow::binary()), arrow::field("f_date", arrow::date32()), + arrow::field("f_time_0", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("f_time_3", arrow::time32(arrow::TimeUnit::MILLI)), arrow::field("f_ts_0", arrow::timestamp(arrow::TimeUnit::SECOND)), arrow::field("f_ts_3", arrow::timestamp(arrow::TimeUnit::MILLI)), arrow::field("f_ts_6", arrow::timestamp(arrow::TimeUnit::MICRO)), @@ -3497,10 +3499,10 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { // from null parents; see the README alongside the Java compatibility table. auto expected = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ -[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1", "bin1", "\u0000\u0001\u0002\u007f", -1, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null, []], [1, "required"], [1, "nullable"], [1.0, -1.5, 0.25]], -[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [null, null], [null, null], null], -[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", "bin1", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [], [[10, null], null, []], [10, "required"], [10, "nullable"], [10.0, -1.5, 0.25]], -[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11", "bin1", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null, []], [11, "required"], [11, "nullable"], [11.0, -1.5, 0.25]] +[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1", "bin1", "\u0000\u0001\u0002\u007f", -1, 0, 86399999, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null, []], [1, "required"], [1, "nullable"], [1.0, -1.5, 0.25]], +[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [null, null], [null, null], null], +[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", "bin1", "\u0000\u0001\u0002\u007f", 20000, 45296000, 1123, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [], [[10, null], null, []], [10, "required"], [10, "nullable"], [10.0, -1.5, 0.25]], +[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11", "bin1", "\u0000\u0001\u0002\u007f", 20000, 86399000, 45296789, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null, []], [11, "required"], [11, "nullable"], [11.0, -1.5, 0.25]] ])") .ValueOrDie()); const std::string table_path = GetDataDir() + "/lance/append_java_compat.db/append_java_compat"; @@ -3546,7 +3548,7 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { auto projected = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields[0], fields[24], fields[1], fields[25]}), R"([ + arrow::struct_({fields[0], fields[26], fields[1], fields[27]}), R"([ [0, [1, "nullable"], 1, [1.0, -1.5, 0.25]], [0, [null, null], 2, null], [0, [10, "nullable"], 10, [10.0, -1.5, 0.25]], @@ -3570,6 +3572,8 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { arrow::field("f_string", arrow::utf8()), arrow::field("f_binary", arrow::binary()), arrow::field("f_date", arrow::date32()), + arrow::field("f_time_0", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("f_time_3", arrow::time32(arrow::TimeUnit::MILLI)), arrow::field("f_ts_0", arrow::timestamp(arrow::TimeUnit::SECOND)), arrow::field("f_ts_3", arrow::timestamp(arrow::TimeUnit::MILLI)), arrow::field("f_ts_6", arrow::timestamp(arrow::TimeUnit::MICRO)), @@ -3586,10 +3590,10 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { }; auto expected = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ -[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", "\u0000\u0001\u0002\u007f", -1, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null, []], [1, "row-1"], [1.0, -1.5, 0.25]], -[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [null, null], null], -[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [], [[10, null], null, []], [10, "row-10"], [10.0, -1.5, 0.25]], -[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null, []], [11, "row-11"], [11.0, -1.5, 0.25]] +[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", "\u0000\u0001\u0002\u007f", -1, 0, 86399999, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null, []], [1, "row-1"], [1.0, -1.5, 0.25]], +[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [null, null], null], +[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "", "\u0000\u0001\u0002\u007f", 20000, 45296000, 1123, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [], [[10, null], null, []], [10, "row-10"], [10.0, -1.5, 0.25]], +[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", "\u0000\u0001\u0002\u007f", 20000, 86399000, 45296789, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null, []], [11, "row-11"], [11.0, -1.5, 0.25]] ])") .ValueOrDie()); const std::string table_path = @@ -3623,7 +3627,7 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { check(2, /*projection=*/{}, expected); auto projected = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields[0], fields[21], fields[1], fields[22], fields[19]}), R"([ + arrow::struct_({fields[0], fields[23], fields[1], fields[24], fields[21]}), R"([ [0, [1, "row-1"], 1, [1.0, -1.5, 0.25], [1, null, -1]], [0, [null, null], 2, null, null], [0, [10, "row-10"], 10, [10.0, -1.5, 0.25], []], diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 441c6d81f..8b36d3c72 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -5978,37 +5978,52 @@ TEST(VortexPrimaryKeyWriteInteTest, TestPKWriteFinalize) { R"([[0, "a", 1],[0, "b", 2]])")); ASSERT_TRUE(success); } +#endif +#if defined(PAIMON_ENABLE_LANCE) || defined(PAIMON_ENABLE_VORTEX) // Regression: append-table finalization always extracts statistics, and TIME has no statistics // factory of its own, so the extractor must still yield one entry for a TIME column. -TEST(VortexTimeColumnWriteInteTest, TestAppendTimeColumnFinalize) { - auto dir = UniqueTestDirectory::Create("local"); - ASSERT_NE(dir, nullptr); - arrow::FieldVector fields = {arrow::field("f_time", arrow::time32(arrow::TimeUnit::MILLI)), - arrow::field("f1", arrow::int32())}; - auto schema = arrow::schema(fields); - std::map options = { - {Options::FILE_FORMAT, "vortex"}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, - /*is_streaming_mode=*/false)); - int64_t commit_identifier = 0; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[3723123, 1],[null, 2]])", - /*partition_map=*/{}, /*bucket=*/0, {})); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - ASSERT_OK_AND_ASSIGN(std::vector> splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), splits, - R"([[0, 3723123, 1],[0, null, 2]])")); - ASSERT_TRUE(success); +TEST(TimeColumnWriteInteTest, TestAppendTimeColumnFinalize) { + std::vector file_formats; +#ifdef PAIMON_ENABLE_LANCE + file_formats.push_back("lance"); +#endif +#ifdef PAIMON_ENABLE_VORTEX + file_formats.push_back("vortex"); +#endif + for (const auto& file_format : file_formats) { + SCOPED_TRACE(file_format); + auto dir = UniqueTestDirectory::Create("local"); + ASSERT_NE(dir, nullptr); + arrow::FieldVector fields = {arrow::field("f_time", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, "local"}}; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[3723123, 1],[null, 2]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN( + bool success, helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), splits, + R"([[0, 3723123, 1],[0, null, 2]])")); + ASSERT_TRUE(success); + } } #endif diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/README.md b/test/test_data/lance/append_java_compat.db/append_java_compat/README.md index b80f6e9a1..71ae1ef9f 100644 --- a/test/test_data/lance/append_java_compat.db/append_java_compat/README.md +++ b/test/test_data/lance/append_java_compat.db/append_java_compat/README.md @@ -16,6 +16,8 @@ f_string STRING f_binary BINARY(4) f_varbinary BYTES f_date DATE +f_time_0 TIME(0) +f_time_3 TIME(3) f_ts_0 TIMESTAMP(0) f_ts_3 TIMESTAMP(3) f_ts_6 TIMESTAMP(6) @@ -41,18 +43,20 @@ BatchTableWrite/BatchTableCommit APIs wrote two rows per commit into two Lance d with schema, Avro manifests and two snapshots. All four rows were read back and verified in Java. Values below follow schema order; timestamps are UTC and binary values are hexadecimal. -Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1", 0x62696e31, 0x0001027f, 1969-12-31, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []], (1, "required"), (1, "nullable"), [1.0, -1.5, 0.25]) -Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), (NULL, NULL), NULL) +Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1", 0x62696e31, 0x0001027f, 1969-12-31, 00:00:00, 23:59:59.999, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []], (1, "required"), (1, "nullable"), [1.0, -1.5, 0.25]) +Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), (NULL, NULL), NULL) Commit - snapshot 1 (2 rows) -Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", 0x62696e31, 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10, "required"), (10, "nullable"), [10.0, -1.5, 0.25]) -Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11", 0x62696e31, 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL, []], (11, "required"), (11, "nullable"), [11.0, -1.5, 0.25]) +Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", 0x62696e31, 0x0001027f, 2024-10-04, 12:34:56, 00:00:01.123, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10, "required"), (10, "nullable"), [10.0, -1.5, 0.25]) +Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11", 0x62696e31, 0x0001027f, 2024-10-04, 23:59:59, 12:34:56.789, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL, []], (11, "required"), (11, "nullable"), [11.0, -1.5, 0.25]) Commit - snapshot 2 (4 rows total) -At this Java revision, nullable ROW declarations are accepted but actual null ROW parents +At the writer commit above, nullable ROW declarations are accepted but actual null ROW parents do not survive the Lance round trip (a separate probe read id=2's null parent as non-null). Both struct parents here are valid, including id=2 where their children are NULL. -TIME32 is omitted because the C++ table path does not support it; Java's Lance validator -rejects local-zoned timestamps, MAP, MULTISET, VARIANT and BLOB. +Java writes both TIME columns to Lance as time32[ms] holding milliseconds of the day, with +no precision in the field metadata; the declared precision comes from the table schema. +At the writer commit above, Java's Lance validator rejects local-zoned timestamps, MAP, +MULTISET, VARIANT and BLOB, so this fixture omits them. diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance similarity index 75% rename from test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance rename to test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance index d95d1d23246280c73ab21a6f31b8405578f9da41..baebf061f979ff9f53b4e848970d02f852150965 100644 GIT binary patch delta 1340 zcmZ{iO-$2Z7{}Xf9iv;jt^@XN9czJ!5s8`W7#qa7MTuYx5~7I^4G!4ikU<#TOf-ZU z6Hgo@Pds_RgC`O%5ECz$8NFyUk&9mNIsp#GWgB3fdDwO*;1%PfjRNn6ZtaxaWn9WY+W>Kra!3R$V9*2yF$3t>` zH{Yu_*IV#HXkQByP3%w7nh8QkX@-)zcV2nBN^VM;FQGQ@vnh)h*&Ji51kC&LmYM^K zz5=YHH^ed0QY9gSu2^b{WMWi^9SaF|=LQdJt(I0>g^O2KGQ zw7Xe6g6t6ClJGnzTHRF!eg$9Sj65ONFhbg@o|*@4G*@6fbV)vEEm~u%UL{Hl$pgIb zg&$b1c(J{!pbGZU#Dh($F{Ak#CJr_m{o5w~YU1%0HGfH!W$c|XfSvOh60mI(4&JQ)-E0C=)uli%EU`ra;k7!q8ZHuvK!I;0Lv66pLu@1H=PYkOUJkYqJhB1s( b7nMd$wSOegu5w>kRa_P|WW~;8hEMzjM|T3G delta 1202 zcmZ9KO-$2p7{=SJgW0ZM|H0O+TUU^9Ks;=NQ4ATPk`Oh-gB&z5rLs*26~k6An&@hb zN8-nO@F zq=00?_%KLJ47QjQJ_!4ah;u-(LsG6)b`dcpwNRWX-_Ue5t*9A+hiO)UJ%)l8EW%Np z;;_voeQ5>u*(4qYno|<#;a0^sqDrb(sc5&=^1LB*fX4O19!t@qsI8}<8Cs9fDR;n2 zu9rw>hcmFnrC^#nE~Q%)`GaIsj`R_FxoW^Fml87r=|j7=`UE-J=xg{xt(KE0SW>G- zWpTz3V(^ylgJq6_*N7zJa;z^!;-E#09$M_6P?8hP!sm#hryAQV6=#f_6(I%3!g*gtS(^)+ z0f6{5JP1$X*WrEa3rvdyPQtcW_H8uAEEL4AfpOAjhJa5l_(LWQ50eIP$x%?GEY5%~ zE%+Ieg`d(U&O%KnwqGC#liT26IkE(g*kitzif8=sf1-Tx&D0Y zlPx$0@<}Uj$u4+iv-1gX$ujI$h~2Z<$$z)m$&aPn`42WP)#Z_fWgf8tyFqI@|HOc7 S@j$mV@%peNKX!g>^7MbNoZ5u| diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance similarity index 78% rename from test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance rename to test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance index d65dffbe82acb8632758845ba2011c89377562bc..22c86f74680a9195a26a65f179acfef632f53751 100644 GIT binary patch delta 1345 zcmZ{iZ%7ky7{~8+Zt8A2_q*Hex81fo=Vl>Nx=o!1rbtC*A%sv_jj0nXr)=IVgylqf zRfPE0?+?ERgD;2f zS(X_K9A@V#w}%_e6h#3|cdix83st5qwANSK#xxv3RpW7GB9)A{V4J*cEV`*$D@MXy zsTo*6;=b^i%CY2lB0H&Ai!Bjc^ZynhQfxVjEb1;Z>K?VQ=xBpM#4!M;u2x;RP3iSG=v_5MsySCLe?APPXPCvINHxN+O6|vHnvf)^a2IkT_e z%lly|G(7(dtDjn36C_ZQn;N_t7A7pN~~ z9QT##A9OYjaMEA8!A2SPN~QYq5}v21{c@?{VHx}CiGG;`YNd>80;T%ZGXA4;M(>|) zAj!@J)cvWidciQ93KE0C8#)^|?hO(BGzrvgoeka5NCe|WZ|H3Jzv-;`&uZ55drBML j>%2tDRF9n`v(+RTf9B#|!q=Keq}4z!%TNQoC;I*ZGW-b2 delta 1215 zcmZ9K%}>){9LL+e)9$q|>$7$1iW^`SUhHtNMS~E+WeC|pj7qf5vFYN5GCF)o2)lY9 zk^Ig^j7JY9?jIQ8%*l%k4Av6FUmyEa|#M*nkjS*Ffg}Bkl&>hS*UG9>E*Lc- zXmF!&k8{B}=ODsjs8hppU01to6vR-w@q{B?E3Nx%74pi9DvH2!G-FDn8eh<{Rs9SS ze*@xT((bIgC{u@FBXVkABrHu&-3!^ln%P_TwI(bLo?90PQ|jrEWC?tz{jW!D4_zCp@(L!$$>|BY0natI%*==8`)}9hL-SwJYlQz zO}-ui@72|v7>Bi=k~Xkiowsy0BW>|Y=eK=)(Z|mMz4?#?NosBnE9MlZM z|Iy9r`B>(P7BDs(=zL$3v0${<-_u!tg_t|0`9}V!&PLy{&PIN9NXx%h$IaGq)~yM@ a9NIMgi8DQTHlpdcI>h$UjNP7Fp8pT^B-&yC diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0 similarity index 91% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0 index 92427a911bddd58beabbd5a6a3b15564ee7426dd..4d08515bf1eb14ef81ee3466bd15072b39359b6a 100644 GIT binary patch delta 150 zcmew+@Ks>L3-n?trSS!{z_Y z4g6KRm!zx<^St#{zY&wv^X;Uf83wprI1;^#&OP_yI*g~ x9N!b7Z delta 151 zcmew=@J(RD3-2$pi| z469{S47+hpD2?yV?zOF3tQE>Di+2Y9uebR(gN3V!k>UU2{tiFqioI)`mc)2%efp!~ v&xD-4vpb_VPtV*Gx?Ub?+eDeQ~@kLLG8^@)bF5Ni0Qr3Lg vly!Cf;zq&O-k5CL!kptR9@%uhc`++PL4*7g=HNEx4~&)$EI_+?Q7i`le_2I$ delta 149 zcmew&@I_$53-bph5l#b8wsU2S!T=7NFg{D3${NJxxQs diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0 similarity index 86% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0 index 961f1f8772838d1a08d958cff7f0b13d19db1d1c..b20eaa6dd906137f499d3b5a66f4793e2b231eda 100644 GIT binary patch delta 110 zcmcb{agAfcJ!XM$3pu^rpLX&&%Q1(0t7JP(e#a~x$lcn`e6Kpc{meoqBYw5s=)NVB znQ~*Vm%g4gD~jVryLZM+j+p88tVVsE=lJ#WD?=8guV9X$bZYlsL C(J+>9p~TLqlix9m2W~!an9=)pX_4QFX}j0T8hk$P z`ABoy```O&rKRIJ<}A%RTzHM5Z~%dH@`WtUPLT*U*8ESs1Z4m>bs5J5K; E02^O1cK`qY diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1 similarity index 86% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1 index 13766e16b12d4d85afe0da3170afd3f2f8174636..77e608f6457326bd3fbbde7aa5a503f39a4c7202 100644 GIT binary patch delta 89 zcmcb~ag$@iJ!S#s>+@0;Z28EyVTF~iXztpq$?upY)lG~nQc?^P&2$aSP0e&oOj3+> fEsZUcb<>P34NWaA%}mmgl64JUOx9zO#xMW?w<#Z` delta 89 zcmcb~ag$@iJ!XMPTgB}9-E&v{X*J>&c^$$$`5m*QdYY+aYFc8NnXXB4Qi`sLsYQ}* fQi@56Zn9Bol7)e#foY0CqOQT4$$Bi(7zO|Ut7;zp diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-0 similarity index 95% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-0 index 96b432730438f76009f498ab22bddb0269b9478d..17888a4631fa5b401ef79e498f5dc37c1f0b3ce6 100644 GIT binary patch delta 24 gcmaFI{*HaaJ!XNP8@D%aNfVnV*Lh{vX$9R?0Gl2RLjV8( delta 24 icmV+z0O$Yi2kr;3-UATIH_my(!^-%pe3QY>`X>OTS`CH( diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-1 similarity index 86% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-1 index 4f1760c6c00de8ae3af71093aec6772f49805bf3..a338618fb202acb037d9f1a5ee11a8fdf3111841 100644 GIT binary patch delta 110 zcmcb{agAfcJ!S!wPba4NGRfQz<8R-mW)l%S`5m)(Aa`p!^S$c)_A?8ejQG`hqx+Uj zX3CAdUiy00tSF8f?cNzPIbx>Uvl{hvp5xcguMAm~!mg90$ff2kz{GIifiZ&!x~TxN CK`N*K delta 110 zcmcb{agAfcJ!XMfR~_&EY%b#1p1~HuJEg05@;henz|98^GkX6nE%G}tZTDJPgU`o3 zA8BrT|9fApv~)bjoTXW(Ty3^zzKBVzW;XI=xfR5=>{4ojt9YQ1WfK#_fd|G6BIu?9 E05>x+p8x;= diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0 index 2d987449f..2be1bf700 100644 --- a/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0 +++ b/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0 @@ -51,45 +51,53 @@ "type" : "DATE" }, { "id" : 12, + "name" : "f_time_0", + "type" : "TIME(0)" + }, { + "id" : 13, + "name" : "f_time_3", + "type" : "TIME(3)" + }, { + "id" : 14, "name" : "f_ts_0", "type" : "TIMESTAMP(0)" }, { - "id" : 13, + "id" : 15, "name" : "f_ts_3", "type" : "TIMESTAMP(3)" }, { - "id" : 14, + "id" : 16, "name" : "f_ts_6", "type" : "TIMESTAMP(6)" }, { - "id" : 15, + "id" : 17, "name" : "f_ts_9", "type" : "TIMESTAMP(9)" }, { - "id" : 16, + "id" : 18, "name" : "f_decimal_1_0", "type" : "DECIMAL(1, 0)" }, { - "id" : 17, + "id" : 19, "name" : "f_decimal_18_2", "type" : "DECIMAL(18, 2)" }, { - "id" : 18, + "id" : 20, "name" : "f_decimal_19_2", "type" : "DECIMAL(19, 2)" }, { - "id" : 19, + "id" : 21, "name" : "f_decimal_38_18", "type" : "DECIMAL(38, 18)" }, { - "id" : 20, + "id" : 22, "name" : "f_array_int", "type" : { "type" : "ARRAY", "element" : "INT" } }, { - "id" : 21, + "id" : 23, "name" : "f_array_array_int", "type" : { "type" : "ARRAY", @@ -99,37 +107,37 @@ } } }, { - "id" : 22, + "id" : 24, "name" : "f_struct", "type" : { "type" : "ROW NOT NULL", "fields" : [ { - "id" : 23, + "id" : 25, "name" : "number", "type" : "INT" }, { - "id" : 24, + "id" : 26, "name" : "label", "type" : "STRING" } ] } }, { - "id" : 25, + "id" : 27, "name" : "f_nullable_struct", "type" : { "type" : "ROW", "fields" : [ { - "id" : 26, + "id" : 28, "name" : "number", "type" : "INT" }, { - "id" : 27, + "id" : 29, "name" : "label", "type" : "STRING" } ] } }, { - "id" : 28, + "id" : 30, "name" : "f_vector", "type" : { "type" : "VECTOR", @@ -137,7 +145,7 @@ "length" : 3 } } ], - "highestFieldId" : 28, + "highestFieldId" : 30, "partitionKeys" : [ ], "primaryKeys" : [ ], "options" : { @@ -146,5 +154,5 @@ "file.format" : "lance", "metadata.stats-mode" : "none" }, - "timeMillis" : 1789217950502 + "timeMillis" : 1793168650637 } \ No newline at end of file diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1 b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1 index 2c51d03de..397bf481d 100644 --- a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1 +++ b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1 @@ -1,17 +1,17 @@ { "version" : 3, - "uuid" : "fb7e0704-7508-46bf-8a23-f4774481757c", + "uuid" : "a5063bd4-985c-4cde-976f-4379dbbc7ceb", "id" : 1, "schemaId" : 0, - "baseManifestList" : "manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0", + "baseManifestList" : "manifest-list-24709406-a874-4961-b200-879ceba4452c-0", "baseManifestListSize" : 1006, - "deltaManifestList" : "manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1", + "deltaManifestList" : "manifest-list-24709406-a874-4961-b200-879ceba4452c-1", "deltaManifestListSize" : 1110, - "commitUser" : "ec878fff-cb67-41b2-a424-5b224e88b514", + "commitUser" : "c29d2747-b5d7-44ef-9b72-8b2fe1ca1d31", "writerVersion" : "java-2.1-SNAPSHOT-63ffd403648e1cf9848e32bb29a398c1b3b16ab1", "commitIdentifier" : 9223372036854775807, "commitKind" : "APPEND", - "timeMillis" : 1789217951996, + "timeMillis" : 1793168652996, "totalRecordCount" : 2, "deltaRecordCount" : 2, "nextRowId" : 0 diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2 b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2 index 3ef365f25..ab9191eb2 100644 --- a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2 +++ b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2 @@ -1,17 +1,17 @@ { "version" : 3, - "uuid" : "9260373b-900c-43b2-b45b-f741cff6fe35", + "uuid" : "e069c81e-0707-43f5-8e9d-027e487bb3b4", "id" : 2, "schemaId" : 0, - "baseManifestList" : "manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0", + "baseManifestList" : "manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0", "baseManifestListSize" : 1110, - "deltaManifestList" : "manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1", + "deltaManifestList" : "manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1", "deltaManifestListSize" : 1113, - "commitUser" : "ede562c1-31df-47bd-b685-36c36f49ef2e", + "commitUser" : "25bacb07-4a95-4cd5-94a3-36a045c65466", "writerVersion" : "java-2.1-SNAPSHOT-63ffd403648e1cf9848e32bb29a398c1b3b16ab1", "commitIdentifier" : 9223372036854775807, "commitKind" : "APPEND", - "timeMillis" : 1789217952098, + "timeMillis" : 1793168653159, "totalRecordCount" : 4, "deltaRecordCount" : 2, "nextRowId" : 0 diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/README.md b/test/test_data/lance/append_python_compat.db/append_python_compat/README.md index bf2f870c1..bddeaee73 100644 --- a/test/test_data/lance/append_python_compat.db/append_python_compat/README.md +++ b/test/test_data/lance/append_python_compat.db/append_python_compat/README.md @@ -21,6 +21,8 @@ f_double DOUBLE f_string STRING f_binary BYTES f_date DATE +f_time_0 TIME(0) +f_time_3 TIME(3) f_ts_0 TIMESTAMP(0) f_ts_3 TIMESTAMP(3) f_ts_6 TIMESTAMP(6) @@ -43,16 +45,20 @@ metadata.stats-mode = none Values below follow schema order; timestamps are UTC and binary values are hexadecimal. The second row has NULL in every field except id and the valid ROW parent with two NULL children. -Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", 0x0001027f, 1969-12-31, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []], (1, "row-1"), [1.0, -1.5, 0.25]) -Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), NULL) +Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", 0x0001027f, 1969-12-31, 00:00:00, 23:59:59.999, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []], (1, "row-1"), [1.0, -1.5, 0.25]) +Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), NULL) Commit - snapshot 1 (2 rows) -Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "", 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10, "row-10"), [10.0, -1.5, 0.25]) -Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL, []], (11, "row-11"), [11.0, -1.5, 0.25]) +Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "", 0x0001027f, 2024-10-04, 12:34:56, 00:00:01.123, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10, "row-10"), [10.0, -1.5, 0.25]) +Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", 0x0001027f, 2024-10-04, 23:59:59, 12:34:56.789, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL, []], (11, "row-11"), [11.0, -1.5, 0.25]) Commit - snapshot 2 (4 rows total) +Paimon Python maps every pyarrow time type to TIME(0), so f_time_3 was declared TIME(3) +directly in the Paimon schema. Paimon Python writes both TIME columns to Lance as time32[ms] +holding milliseconds of the day, with no field metadata; the declared precision comes from +the table schema. Null and empty lists, null list elements, an empty string, binary zero bytes, null scalars, null ROW children and a null vector are included. ROW parents are always valid: this fixture does not claim null-parent ROW compatibility. diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-24cc7753-16ed-451d-8899-984adb2de133-0.lance similarity index 71% rename from test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance rename to test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-24cc7753-16ed-451d-8899-984adb2de133-0.lance index 37aaea1d9be609a5781f3d226fc5aebdfaf61971..693e37e90e330207dda12b61d9da50755792ac8f 100644 GIT binary patch delta 948 zcma)(O=uHA6vuaVl5{h($tIg*v-wcA6fLbn)2@fK)sP}Z47(~}@mP#|4n%tv>cxZAf=5ptg?i9QXBNAA>Kx|(e((Q2U}kUusy(c9raala zeKy)%szMz{@b-DHTc3ZAKSbnj#M>SFh?+CH;;gjSnvIo;NvyyALDTV3(wBD|v({GtlZSy^~Rr;!1DI*Id^0sFL!O)C$FbQ~3+tWM#gRe-DNXP`e?&4FHeqz{1Vd&(sucNCp%{+zO*r_GEK!{Z6)XTC&htct*R))_^^ zQM8UIY-S%}+EKT1ZKN`&ftRym2_03;7<9AqaK^B4qLYMs#wV2A(i-#ziFCA~rpqZ6 zzG};Gke_9nQ|7Y!CLtZ2H7*iK2Dv48UHIQad`07i@ApGoN(KFwf}iuWE6HgdiG<_| z_6Pn9@l8EAa1i3jOwjL!_)macNor(y!$U68UWgZSLH})t(+2k+i~bjap9ww;`5XCT z^=!X%AQFDVEt41A5d(Gv`!8Y^xZ%Ipw&16tzOL|KHOkfhVp`<9U*d|GsHcu%EL~f= Ge)%7$pr4!o delta 763 zcmZ9JJxeP=7=>pt8xwbDqZyOkP5fZ3)Y$k{_?28l1A-zp7Ge>&m#B%T7=wZ!*~T9r zFP5TUuSLL0F<0Ssa?w`7KOmA)6x2;-w%Fa~oadaGmzh}@z8*RyWsCfkd>@VF$>1$k z`dPzrrnsHyGh`^p16To7(cwyN!?fLaP?ESA?vxY`gO8431}WM_ zT1*D3GyzF!D(y_l6|?I`)QrG2Z6lf)g=gA{b@-w^#4vS8s2wC`>tUm&CRuPfvy=T( z+$|RD^v-=Ol;4-#lKj(bVE`7%W*;7s+$i5=SBbN z;cSQ(PXFJ-BTer9Oz=LBOUP{IP6X&!urn|ab`RL{@RNtr%)S0X@EVU%yoDRiC4>%K z0YX&|FGcuH;&j0`b diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance similarity index 75% rename from test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance rename to test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance index a1cc70ab502352bc5bb22674dd9c44f92f9edbe4..0891df17745414a8eee5aa095fa3ea11cc679e15 100644 GIT binary patch delta 935 zcma)*J8aWH7{~8&P13rKeX-+L;ucp(AhM9hLd&C|6eJ!(tPrF&Qd_AKDTJmiiiBbo z78d9?bU{@+A;g%mi^arJF)$z@5G-Y2KrpcpF1fRiJ@p3vzu))&92xo7^HP zxFU7Z{rZU{LI^OP?gb-H(uvPfuO`36&C}ZG@_c)xxinw0C9fYITpSxDtCgANS_z-Q zchZYX%Z>GWZRvkQ)gC`eLs?6)4Qp(^3{D<@&kXc5UDhaIE6W&KEBlSYD!ufVL{CW@~zeE^^AU+L>CpY=8PZLJf{ z*O|7Gm!(9Y!+WPSmJ4!P0?RgmZ`o=1Vw4A-Z!oVptTLcjXc03oVW)6FtiXa*w@vK8 zhC{cLbOOiE!{S7}yQi_MU}uF3l5`$y2=)bU3jP!0)f_iM|1idPji~=Y@ME6tqFW}9 zL;@WM4hPOz(ETgH5i;$A%1&_F(r!oE;?MM3}5jIpNdBJTi=$YWqPaNY$_{F+{pNsm786GqSe;GUB6|3G5(bp-i On2F4H6LjO!&DuY>KcRyF delta 795 zcmZ9J&ubGw9K~mLc2m1QlAX!!&Ngn6^-{4H8xMlkB!*fP6!BU_1C^GR(ALDX6+yEg z;6EQAE7jgSyR3pt}ppd!P3nvoQPo!kgwxZ70_nQ})Mq zXRR;TsIlBqyML!W?d#AhoWcfJHir*|0GmViJ8@Rx&f+{=vd`fHJhJQ9hOc%5Q&8v( zX3(I=Nik%wOsAkuL!->pmHy(Y?}iS1qyh0l7rxVDn8ROsg7_haDOMp9i9X4DR-GML zYHu#y>aX|w((W2FHF)iGi5?m7%IU*`|6}a3uEtiBhgNBW2IyBhKpi)I z;9iF3y>$OA!xwqF|0MaYh|y_Z7{LRC-lYPBCMOOL5PBiGCl9bo!i;X@vE)(zC&Qhi z>HVJM`GdHVyfZ<1up?Ps5jt5G{k9Nv-W2AaDs=PERpdAk@hyU`w64zn2jKT` AIRF3v diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0 similarity index 92% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0 index 27f94daba74adf1ea75136611d54784b71460d74..bb466bdda310ff7cdbc076864695111ccd83cc7b 100644 GIT binary patch delta 150 zcmZ1=xKMC|2uHmTo1OjdKammA--F}&xYz7wy27ZjRsXNTdR}IRCoC)s5)6zsXl?uCTB$NH8!ar6iUl z>Kd6OC!3p_8tWRGrKaebm>Q<&T3A?G>RMWuB&H-8rKB1f8|xbA oa=C5th72YK1|}c`0yc&O0gDCvpBgorWq<C@=x-=0&j_0LaKLd;kCd delta 149 zcmZ1=xIl1&2uJ<%y_+wuIDN%jqFUMK{u|u~OqUonw(9>?SjWoDaD|11L4tuXDJ8Ka zQP;vW(Jak8(M&hl%s5Tg#MCfNH_;@;T-P|+(!|`z#3U`vB30KwFDEfCIrUct3lkGl o+n1fKQ!OV diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0 similarity index 96% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0 index 87d718a252e84fffcd2468e7da264f3a8511b216..d33af15b8e7de0affd79637d1c2fc3b2c02a8863 100644 GIT binary patch delta 24 gcmbQwHlJ++KeND|)1fhIT8`OlWSc3v=G?`N0C#>0=l}o! delta 24 gcmbQwHlJ++KeNF7c_&$%A7?bbzviD&w(Lg=0DT(^YybcN diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1 similarity index 85% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1 index 9b5cf56f58c1cf597c9def10f39c1331ac5d3dce..4e3ebb9dd2040d92749b365fcc93cb3feba170f9 100644 GIT binary patch delta 102 zcmZo`$)+i~ r7UqVQx``%+hRK#j2C0Uorn&})l$aPK7?>D delta 102 zcmZo6#j+80nfMo2KYm qm>XK^CYl%;CR-XAq#Bx<>KYtUVq%bBU}69QCI$n33uXsKbkhL6iyO}X delta 102 zcmZo=tZQisButVm(}1kh oR9%DQK`5)4cXK)}Rcz;D6qz=&=d08JwvGynhq diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1 similarity index 85% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1 index 4ead24d61538a51f00c5d5e6fa0c6fc0872766ba..28c5a85760dde0521a67b7f1aeab462bb37bf3ec 100644 GIT binary patch delta 102 zcmZosfn>} rQnHz)uA!-sxtVFQVX~=ZvaZ1aB_;+51||j|U}7-fw_tW)L^llpq+%d8 delta 102 zcmZo