diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index 34818ea49..92b1dda11 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -146,13 +146,29 @@ and `Arrow DataTypes expected_errors = {"type MAP", "LOCAL_ZONED_TIMESTAMP", "type VARIANT", - "type time32", "type time32"}; + std::vector expected_errors = {"type MAP", "LOCAL_ZONED_TIMESTAMP", + "type VARIANT"}; for (size_t i = 0; i < unsupported_fields.size(); ++i) { ASSERT_OK_AND_ASSIGN( table_schema, diff --git a/src/paimon/format/lance/lance_file_format_test.cpp b/src/paimon/format/lance/lance_file_format_test.cpp index 089f5f328..0805d49e1 100644 --- a/src/paimon/format/lance/lance_file_format_test.cpp +++ b/src/paimon/format/lance/lance_file_format_test.cpp @@ -29,6 +29,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" @@ -176,6 +177,11 @@ TEST(LanceUtilsTest, ConvertsObjectStoreOptions) { } TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { + auto time_field = [](const std::string& name, int32_t precision) { + return arrow::field( + name, arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true, + arrow::key_value_metadata({DataType::kTimePrecision}, {std::to_string(precision)})); + }; arrow::FieldVector fields = { arrow::field("bool_col", arrow::boolean()), arrow::field("i8_col", arrow::int8()), @@ -187,6 +193,7 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { arrow::field("string_col", arrow::utf8()), arrow::field("binary_col", arrow::binary()), arrow::field("date_col", arrow::date32()), + time_field("time_col", 3), arrow::field("timestamp_col", arrow::timestamp(arrow::TimeUnit::MICRO)), arrow::field("decimal_col", arrow::decimal128(12, 2)), arrow::field("array_col", arrow::list(arrow::float32())), @@ -194,15 +201,22 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { arrow::field( "vector_col", arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 2)), + arrow::field("time_array_col", arrow::list(time_field("item", 0))), + arrow::field("time_row_col", arrow::struct_({time_field("value", 6)})), + arrow::field("time_row_array_col", + arrow::struct_({arrow::field("values", arrow::list(time_field("item", 9)))})), }; std::shared_ptr schema = arrow::schema(fields); std::shared_ptr expected = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ - [true,-1,-2,-3,-4,1.5,2.5,"one","abc",1, - "1970-01-01 00:00:00.000001","12.34",[1.0,2.0],[7],[3.0,4.0]], - [false,1,2,3,4,3.5,4.5,"two","xyz",2, - "2030-12-31 23:59:59.999999","-12.34",[],[8],[5.0,6.0]], - [null,null,null,null,null,null,null,null,null,null,null,null,null,[null],null] + [true,-1,-2,-3,-4,1.5,2.5,"one","abc",1,0, + "1970-01-01 00:00:00.000001","12.34",[1.0,2.0],[7],[3.0,4.0], + [0,null,45296000],[45296123],[[3723123,null]]], + [false,1,2,3,4,3.5,4.5,"two","xyz",2,86399999, + "2030-12-31 23:59:59.999999","-12.34",[],[8],[5.0,6.0], + [],[86399999],[[]]], + [null,null,null,null,null,null,null,null,null,null,null,null,null,null,[null],null, + null,[null],[null]] ])") .ValueOrDie(); std::string path = PathUtil::JoinPath(directory_->Str(), "supported-types.lance"); @@ -219,6 +233,19 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) { ASSERT_TRUE(actual->Equals(arrow::ChunkedArray(expected))) << actual->ToString() << "\nvs\n" << expected->ToString(); } + + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, OpenReader(path, 2)); + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&ffi_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + arrow::Result> batch_result = + arrow::ImportArray(batch.first.get(), batch.second.get()); + ASSERT_TRUE(batch_result.ok()) << batch_result.status().ToString(); + std::shared_ptr batch_array = std::move(batch_result).MoveValueUnsafe(); + ASSERT_TRUE(batch_array->type()->Equals(arrow::struct_(fields), /*check_metadata=*/true)) + << batch_array->type()->ToString(/*show_metadata=*/true); } TEST_F(LanceFileFormatTest, RejectsNullTopLevelRows) { @@ -351,10 +378,12 @@ TEST_F(LanceFileFormatTest, EmptyProjectionAndEmptySelection) { TEST_F(LanceFileFormatTest, ExtractStatistics) { arrow::FieldVector fields = {arrow::field("id", arrow::int32()), - arrow::field("name", arrow::utf8())}; + arrow::field("name", arrow::utf8()), + arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI))}; std::shared_ptr schema = arrow::schema(fields); std::shared_ptr data = - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([[1,"a"],[2,null]])") + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([[1,"a",3723123],[2,null,null]])") .ValueOrDie(); std::string path = PathUtil::JoinPath(directory_->Str(), "stats.lance"); ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2)); @@ -365,9 +394,10 @@ TEST_F(LanceFileFormatTest, ExtractStatistics) { format_->CreateStatsExtractor(&ffi_schema)); ASSERT_OK_AND_ASSIGN(auto result, extractor->ExtractWithFileInfo(file_system_, path, pool_)); ASSERT_EQ(result.second.GetRowCount(), 2); - ASSERT_EQ(result.first.size(), 2); + ASSERT_EQ(result.first.size(), 3); ASSERT_EQ(result.first[0]->ToString(), "min null, max null, null count null"); ASSERT_EQ(result.first[1]->ToString(), "min null, max null, null count null"); + ASSERT_EQ(result.first[2]->ToString(), "min null, max null, null count null"); } TEST_F(LanceFileFormatTest, RejectsIncompatibleReadTypes) { diff --git a/src/paimon/format/lance/lance_stats_extractor.cpp b/src/paimon/format/lance/lance_stats_extractor.cpp index 0bd39f298..833af6a59 100644 --- a/src/paimon/format/lance/lance_stats_extractor.cpp +++ b/src/paimon/format/lance/lance_stats_extractor.cpp @@ -86,6 +86,8 @@ Result> LanceStatsExtractor::CreateEmptyStats( return ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::DATE32: return ColumnStats::CreateDateColumnStats(std::nullopt, std::nullopt, std::nullopt); + case arrow::Type::TIME32: + return ColumnStats::CreateIntColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::TIMESTAMP: { std::shared_ptr timestamp_type = checked_pointer_cast(type); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index b7cc73b96..3bfb58c7e 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -3479,6 +3479,8 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { arrow::field("f_binary", arrow::binary()), arrow::field("f_varbinary", arrow::binary()), arrow::field("f_date", arrow::date32()), + arrow::field("f_time_0", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("f_time_3", arrow::time32(arrow::TimeUnit::MILLI)), arrow::field("f_ts_0", arrow::timestamp(arrow::TimeUnit::SECOND)), arrow::field("f_ts_3", arrow::timestamp(arrow::TimeUnit::MILLI)), arrow::field("f_ts_6", arrow::timestamp(arrow::TimeUnit::MICRO)), @@ -3497,10 +3499,10 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { // from null parents; see the README alongside the Java compatibility table. auto expected = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ -[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1", "bin1", "\u0000\u0001\u0002\u007f", -1, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null, []], [1, "required"], [1, "nullable"], [1.0, -1.5, 0.25]], -[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [null, null], [null, null], null], -[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", "bin1", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [], [[10, null], null, []], [10, "required"], [10, "nullable"], [10.0, -1.5, 0.25]], -[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11", "bin1", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null, []], [11, "required"], [11, "nullable"], [11.0, -1.5, 0.25]] +[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1", "bin1", "\u0000\u0001\u0002\u007f", -1, 0, 86399999, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null, []], [1, "required"], [1, "nullable"], [1.0, -1.5, 0.25]], +[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [null, null], [null, null], null], +[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", "bin1", "\u0000\u0001\u0002\u007f", 20000, 45296000, 1123, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [], [[10, null], null, []], [10, "required"], [10, "nullable"], [10.0, -1.5, 0.25]], +[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11", "bin1", "\u0000\u0001\u0002\u007f", 20000, 86399000, 45296789, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null, []], [11, "required"], [11, "nullable"], [11.0, -1.5, 0.25]] ])") .ValueOrDie()); const std::string table_path = GetDataDir() + "/lance/append_java_compat.db/append_java_compat"; @@ -3546,7 +3548,7 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) { auto projected = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields[0], fields[24], fields[1], fields[25]}), R"([ + arrow::struct_({fields[0], fields[26], fields[1], fields[27]}), R"([ [0, [1, "nullable"], 1, [1.0, -1.5, 0.25]], [0, [null, null], 2, null], [0, [10, "nullable"], 10, [10.0, -1.5, 0.25]], @@ -3570,6 +3572,8 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { arrow::field("f_string", arrow::utf8()), arrow::field("f_binary", arrow::binary()), arrow::field("f_date", arrow::date32()), + arrow::field("f_time_0", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("f_time_3", arrow::time32(arrow::TimeUnit::MILLI)), arrow::field("f_ts_0", arrow::timestamp(arrow::TimeUnit::SECOND)), arrow::field("f_ts_3", arrow::timestamp(arrow::TimeUnit::MILLI)), arrow::field("f_ts_6", arrow::timestamp(arrow::TimeUnit::MICRO)), @@ -3586,10 +3590,10 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { }; auto expected = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ -[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", "\u0000\u0001\u0002\u007f", -1, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null, []], [1, "row-1"], [1.0, -1.5, 0.25]], -[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [null, null], null], -[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [], [[10, null], null, []], [10, "row-10"], [10.0, -1.5, 0.25]], -[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null, []], [11, "row-11"], [11.0, -1.5, 0.25]] +[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", "\u0000\u0001\u0002\u007f", -1, 0, 86399999, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null, []], [1, "row-1"], [1.0, -1.5, 0.25]], +[0, 2, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [null, null], null], +[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "", "\u0000\u0001\u0002\u007f", 20000, 45296000, 1123, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [], [[10, null], null, []], [10, "row-10"], [10.0, -1.5, 0.25]], +[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", "\u0000\u0001\u0002\u007f", 20000, 86399000, 45296789, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789", "-3", "1.25", "12345678901234567.89", "12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null, []], [11, "row-11"], [11.0, -1.5, 0.25]] ])") .ValueOrDie()); const std::string table_path = @@ -3623,7 +3627,7 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility) { check(2, /*projection=*/{}, expected); auto projected = std::make_shared( arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields[0], fields[21], fields[1], fields[22], fields[19]}), R"([ + arrow::struct_({fields[0], fields[23], fields[1], fields[24], fields[21]}), R"([ [0, [1, "row-1"], 1, [1.0, -1.5, 0.25], [1, null, -1]], [0, [null, null], 2, null, null], [0, [10, "row-10"], 10, [10.0, -1.5, 0.25], []], diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 441c6d81f..8b36d3c72 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -5978,37 +5978,52 @@ TEST(VortexPrimaryKeyWriteInteTest, TestPKWriteFinalize) { R"([[0, "a", 1],[0, "b", 2]])")); ASSERT_TRUE(success); } +#endif +#if defined(PAIMON_ENABLE_LANCE) || defined(PAIMON_ENABLE_VORTEX) // Regression: append-table finalization always extracts statistics, and TIME has no statistics // factory of its own, so the extractor must still yield one entry for a TIME column. -TEST(VortexTimeColumnWriteInteTest, TestAppendTimeColumnFinalize) { - auto dir = UniqueTestDirectory::Create("local"); - ASSERT_NE(dir, nullptr); - arrow::FieldVector fields = {arrow::field("f_time", arrow::time32(arrow::TimeUnit::MILLI)), - arrow::field("f1", arrow::int32())}; - auto schema = arrow::schema(fields); - std::map options = { - {Options::FILE_FORMAT, "vortex"}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, - /*is_streaming_mode=*/false)); - int64_t commit_identifier = 0; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[3723123, 1],[null, 2]])", - /*partition_map=*/{}, /*bucket=*/0, {})); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - ASSERT_OK_AND_ASSIGN(std::vector> splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), splits, - R"([[0, 3723123, 1],[0, null, 2]])")); - ASSERT_TRUE(success); +TEST(TimeColumnWriteInteTest, TestAppendTimeColumnFinalize) { + std::vector file_formats; +#ifdef PAIMON_ENABLE_LANCE + file_formats.push_back("lance"); +#endif +#ifdef PAIMON_ENABLE_VORTEX + file_formats.push_back("vortex"); +#endif + for (const auto& file_format : file_formats) { + SCOPED_TRACE(file_format); + auto dir = UniqueTestDirectory::Create("local"); + ASSERT_NE(dir, nullptr); + arrow::FieldVector fields = {arrow::field("f_time", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, "local"}}; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[3723123, 1],[null, 2]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN( + bool success, helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), splits, + R"([[0, 3723123, 1],[0, null, 2]])")); + ASSERT_TRUE(success); + } } #endif diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/README.md b/test/test_data/lance/append_java_compat.db/append_java_compat/README.md index b80f6e9a1..71ae1ef9f 100644 --- a/test/test_data/lance/append_java_compat.db/append_java_compat/README.md +++ b/test/test_data/lance/append_java_compat.db/append_java_compat/README.md @@ -16,6 +16,8 @@ f_string STRING f_binary BINARY(4) f_varbinary BYTES f_date DATE +f_time_0 TIME(0) +f_time_3 TIME(3) f_ts_0 TIMESTAMP(0) f_ts_3 TIMESTAMP(3) f_ts_6 TIMESTAMP(6) @@ -41,18 +43,20 @@ BatchTableWrite/BatchTableCommit APIs wrote two rows per commit into two Lance d with schema, Avro manifests and two snapshots. All four rows were read back and verified in Java. Values below follow schema order; timestamps are UTC and binary values are hexadecimal. -Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1", 0x62696e31, 0x0001027f, 1969-12-31, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []], (1, "required"), (1, "nullable"), [1.0, -1.5, 0.25]) -Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), (NULL, NULL), NULL) +Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1", 0x62696e31, 0x0001027f, 1969-12-31, 00:00:00, 23:59:59.999, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []], (1, "required"), (1, "nullable"), [1.0, -1.5, 0.25]) +Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), (NULL, NULL), NULL) Commit - snapshot 1 (2 rows) -Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", 0x62696e31, 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10, "required"), (10, "nullable"), [10.0, -1.5, 0.25]) -Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11", 0x62696e31, 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL, []], (11, "required"), (11, "nullable"), [11.0, -1.5, 0.25]) +Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", 0x62696e31, 0x0001027f, 2024-10-04, 12:34:56, 00:00:01.123, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10, "required"), (10, "nullable"), [10.0, -1.5, 0.25]) +Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11", 0x62696e31, 0x0001027f, 2024-10-04, 23:59:59, 12:34:56.789, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL, []], (11, "required"), (11, "nullable"), [11.0, -1.5, 0.25]) Commit - snapshot 2 (4 rows total) -At this Java revision, nullable ROW declarations are accepted but actual null ROW parents +At the writer commit above, nullable ROW declarations are accepted but actual null ROW parents do not survive the Lance round trip (a separate probe read id=2's null parent as non-null). Both struct parents here are valid, including id=2 where their children are NULL. -TIME32 is omitted because the C++ table path does not support it; Java's Lance validator -rejects local-zoned timestamps, MAP, MULTISET, VARIANT and BLOB. +Java writes both TIME columns to Lance as time32[ms] holding milliseconds of the day, with +no precision in the field metadata; the declared precision comes from the table schema. +At the writer commit above, Java's Lance validator rejects local-zoned timestamps, MAP, +MULTISET, VARIANT and BLOB, so this fixture omits them. diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance similarity index 75% rename from test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance rename to test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance index d95d1d232..baebf061f 100644 Binary files a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance and b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance differ diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance similarity index 78% rename from test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance rename to test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance index d65dffbe8..22c86f746 100644 Binary files a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance and b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance differ diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0 similarity index 91% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0 index 92427a911..4d08515bf 100644 Binary files a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0 and b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0 differ diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-ce7a697c-635e-420f-8014-af0166fa5eea-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-428dd1a6-0756-44d3-939c-f39159964fbc-0 similarity index 91% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-ce7a697c-635e-420f-8014-af0166fa5eea-0 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-428dd1a6-0756-44d3-939c-f39159964fbc-0 index f208ef70d..c57028159 100644 Binary files a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-ce7a697c-635e-420f-8014-af0166fa5eea-0 and b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-428dd1a6-0756-44d3-939c-f39159964fbc-0 differ diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0 similarity index 86% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0 index 961f1f877..b20eaa6dd 100644 Binary files a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0 and b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0 differ diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1 similarity index 86% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1 index 13766e16b..77e608f64 100644 Binary files a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1 and b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1 differ diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-0 similarity index 95% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-0 index 96b432730..17888a463 100644 Binary files a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0 and b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-0 differ diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1 b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-1 similarity index 86% rename from test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1 rename to test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-1 index 4f1760c6c..a338618fb 100644 Binary files a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1 and b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-1 differ diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0 b/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0 index 2d987449f..2be1bf700 100644 --- a/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0 +++ b/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0 @@ -51,45 +51,53 @@ "type" : "DATE" }, { "id" : 12, + "name" : "f_time_0", + "type" : "TIME(0)" + }, { + "id" : 13, + "name" : "f_time_3", + "type" : "TIME(3)" + }, { + "id" : 14, "name" : "f_ts_0", "type" : "TIMESTAMP(0)" }, { - "id" : 13, + "id" : 15, "name" : "f_ts_3", "type" : "TIMESTAMP(3)" }, { - "id" : 14, + "id" : 16, "name" : "f_ts_6", "type" : "TIMESTAMP(6)" }, { - "id" : 15, + "id" : 17, "name" : "f_ts_9", "type" : "TIMESTAMP(9)" }, { - "id" : 16, + "id" : 18, "name" : "f_decimal_1_0", "type" : "DECIMAL(1, 0)" }, { - "id" : 17, + "id" : 19, "name" : "f_decimal_18_2", "type" : "DECIMAL(18, 2)" }, { - "id" : 18, + "id" : 20, "name" : "f_decimal_19_2", "type" : "DECIMAL(19, 2)" }, { - "id" : 19, + "id" : 21, "name" : "f_decimal_38_18", "type" : "DECIMAL(38, 18)" }, { - "id" : 20, + "id" : 22, "name" : "f_array_int", "type" : { "type" : "ARRAY", "element" : "INT" } }, { - "id" : 21, + "id" : 23, "name" : "f_array_array_int", "type" : { "type" : "ARRAY", @@ -99,37 +107,37 @@ } } }, { - "id" : 22, + "id" : 24, "name" : "f_struct", "type" : { "type" : "ROW NOT NULL", "fields" : [ { - "id" : 23, + "id" : 25, "name" : "number", "type" : "INT" }, { - "id" : 24, + "id" : 26, "name" : "label", "type" : "STRING" } ] } }, { - "id" : 25, + "id" : 27, "name" : "f_nullable_struct", "type" : { "type" : "ROW", "fields" : [ { - "id" : 26, + "id" : 28, "name" : "number", "type" : "INT" }, { - "id" : 27, + "id" : 29, "name" : "label", "type" : "STRING" } ] } }, { - "id" : 28, + "id" : 30, "name" : "f_vector", "type" : { "type" : "VECTOR", @@ -137,7 +145,7 @@ "length" : 3 } } ], - "highestFieldId" : 28, + "highestFieldId" : 30, "partitionKeys" : [ ], "primaryKeys" : [ ], "options" : { @@ -146,5 +154,5 @@ "file.format" : "lance", "metadata.stats-mode" : "none" }, - "timeMillis" : 1789217950502 + "timeMillis" : 1793168650637 } \ No newline at end of file diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1 b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1 index 2c51d03de..397bf481d 100644 --- a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1 +++ b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1 @@ -1,17 +1,17 @@ { "version" : 3, - "uuid" : "fb7e0704-7508-46bf-8a23-f4774481757c", + "uuid" : "a5063bd4-985c-4cde-976f-4379dbbc7ceb", "id" : 1, "schemaId" : 0, - "baseManifestList" : "manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0", + "baseManifestList" : "manifest-list-24709406-a874-4961-b200-879ceba4452c-0", "baseManifestListSize" : 1006, - "deltaManifestList" : "manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1", + "deltaManifestList" : "manifest-list-24709406-a874-4961-b200-879ceba4452c-1", "deltaManifestListSize" : 1110, - "commitUser" : "ec878fff-cb67-41b2-a424-5b224e88b514", + "commitUser" : "c29d2747-b5d7-44ef-9b72-8b2fe1ca1d31", "writerVersion" : "java-2.1-SNAPSHOT-63ffd403648e1cf9848e32bb29a398c1b3b16ab1", "commitIdentifier" : 9223372036854775807, "commitKind" : "APPEND", - "timeMillis" : 1789217951996, + "timeMillis" : 1793168652996, "totalRecordCount" : 2, "deltaRecordCount" : 2, "nextRowId" : 0 diff --git a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2 b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2 index 3ef365f25..ab9191eb2 100644 --- a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2 +++ b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2 @@ -1,17 +1,17 @@ { "version" : 3, - "uuid" : "9260373b-900c-43b2-b45b-f741cff6fe35", + "uuid" : "e069c81e-0707-43f5-8e9d-027e487bb3b4", "id" : 2, "schemaId" : 0, - "baseManifestList" : "manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0", + "baseManifestList" : "manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0", "baseManifestListSize" : 1110, - "deltaManifestList" : "manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1", + "deltaManifestList" : "manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1", "deltaManifestListSize" : 1113, - "commitUser" : "ede562c1-31df-47bd-b685-36c36f49ef2e", + "commitUser" : "25bacb07-4a95-4cd5-94a3-36a045c65466", "writerVersion" : "java-2.1-SNAPSHOT-63ffd403648e1cf9848e32bb29a398c1b3b16ab1", "commitIdentifier" : 9223372036854775807, "commitKind" : "APPEND", - "timeMillis" : 1789217952098, + "timeMillis" : 1793168653159, "totalRecordCount" : 4, "deltaRecordCount" : 2, "nextRowId" : 0 diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/README.md b/test/test_data/lance/append_python_compat.db/append_python_compat/README.md index bf2f870c1..bddeaee73 100644 --- a/test/test_data/lance/append_python_compat.db/append_python_compat/README.md +++ b/test/test_data/lance/append_python_compat.db/append_python_compat/README.md @@ -21,6 +21,8 @@ f_double DOUBLE f_string STRING f_binary BYTES f_date DATE +f_time_0 TIME(0) +f_time_3 TIME(3) f_ts_0 TIMESTAMP(0) f_ts_3 TIMESTAMP(3) f_ts_6 TIMESTAMP(6) @@ -43,16 +45,20 @@ metadata.stats-mode = none Values below follow schema order; timestamps are UTC and binary values are hexadecimal. The second row has NULL in every field except id and the valid ROW parent with two NULL children. -Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", 0x0001027f, 1969-12-31, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []], (1, "row-1"), [1.0, -1.5, 0.25]) -Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), NULL) +Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", 0x0001027f, 1969-12-31, 00:00:00, 23:59:59.999, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []], (1, "row-1"), [1.0, -1.5, 0.25]) +Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), NULL) Commit - snapshot 1 (2 rows) -Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "", 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10, "row-10"), [10.0, -1.5, 0.25]) -Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL, []], (11, "row-11"), [11.0, -1.5, 0.25]) +Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "", 0x0001027f, 2024-10-04, 12:34:56, 00:00:01.123, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10, "row-10"), [10.0, -1.5, 0.25]) +Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", 0x0001027f, 2024-10-04, 23:59:59, 12:34:56.789, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL, []], (11, "row-11"), [11.0, -1.5, 0.25]) Commit - snapshot 2 (4 rows total) +Paimon Python maps every pyarrow time type to TIME(0), so f_time_3 was declared TIME(3) +directly in the Paimon schema. Paimon Python writes both TIME columns to Lance as time32[ms] +holding milliseconds of the day, with no field metadata; the declared precision comes from +the table schema. Null and empty lists, null list elements, an empty string, binary zero bytes, null scalars, null ROW children and a null vector are included. ROW parents are always valid: this fixture does not claim null-parent ROW compatibility. diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-24cc7753-16ed-451d-8899-984adb2de133-0.lance similarity index 71% rename from test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance rename to test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-24cc7753-16ed-451d-8899-984adb2de133-0.lance index 37aaea1d9..693e37e90 100644 Binary files a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance and b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-24cc7753-16ed-451d-8899-984adb2de133-0.lance differ diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance similarity index 75% rename from test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance rename to test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance index a1cc70ab5..0891df177 100644 Binary files a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance and b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance differ diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0 similarity index 92% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0 index 27f94daba..bb466bdda 100644 Binary files a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0 and b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0 differ diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-6b2e90c5-eabd-41e9-9a02-099ecdb7fede-0 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-a10d7552-9602-4543-bc69-152765c1c59c-0 similarity index 92% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-6b2e90c5-eabd-41e9-9a02-099ecdb7fede-0 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-a10d7552-9602-4543-bc69-152765c1c59c-0 index 28895ae57..708ee6252 100644 Binary files a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-6b2e90c5-eabd-41e9-9a02-099ecdb7fede-0 and b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-a10d7552-9602-4543-bc69-152765c1c59c-0 differ diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0 similarity index 96% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0 index 87d718a25..d33af15b8 100644 Binary files a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0 and b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0 differ diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1 similarity index 85% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1 index 9b5cf56f5..4e3ebb9dd 100644 Binary files a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1 and b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1 differ diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0 similarity index 85% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0 index 98c1af25b..0b356840c 100644 Binary files a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0 and b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0 differ diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1 b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1 similarity index 85% rename from test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1 rename to test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1 index 4ead24d61..28c5a8576 100644 Binary files a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1 and b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1 differ diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/schema/schema-0 b/test/test_data/lance/append_python_compat.db/append_python_compat/schema/schema-0 index 05be14c14..7243667d9 100644 --- a/test/test_data/lance/append_python_compat.db/append_python_compat/schema/schema-0 +++ b/test/test_data/lance/append_python_compat.db/append_python_compat/schema/schema-0 @@ -54,46 +54,56 @@ }, { "id": 10, + "name": "f_time_0", + "type": "TIME(0)" + }, + { + "id": 11, + "name": "f_time_3", + "type": "TIME(3)" + }, + { + "id": 12, "name": "f_ts_0", "type": "TIMESTAMP(0)" }, { - "id": 11, + "id": 13, "name": "f_ts_3", "type": "TIMESTAMP(3)" }, { - "id": 12, + "id": 14, "name": "f_ts_6", "type": "TIMESTAMP(6)" }, { - "id": 13, + "id": 15, "name": "f_ts_9", "type": "TIMESTAMP(9)" }, { - "id": 14, + "id": 16, "name": "f_decimal_1_0", "type": "DECIMAL(1, 0)" }, { - "id": 15, + "id": 17, "name": "f_decimal_18_2", "type": "DECIMAL(18, 2)" }, { - "id": 16, + "id": 18, "name": "f_decimal_19_2", "type": "DECIMAL(19, 2)" }, { - "id": 17, + "id": 19, "name": "f_decimal_38_18", "type": "DECIMAL(38, 18)" }, { - "id": 18, + "id": 20, "name": "f_array_int", "type": { "type": "ARRAY", @@ -102,7 +112,7 @@ } }, { - "id": 19, + "id": 21, "name": "f_array_array_int", "type": { "type": "ARRAY", @@ -115,18 +125,18 @@ } }, { - "id": 20, + "id": 22, "name": "f_struct", "type": { "type": "ROW", "fields": [ { - "id": 21, + "id": 23, "name": "number", "type": "INT" }, { - "id": 22, + "id": 24, "name": "label", "type": "STRING" } @@ -135,7 +145,7 @@ } }, { - "id": 23, + "id": 25, "name": "f_vector", "type": { "type": "VECTOR", @@ -145,7 +155,7 @@ } } ], - "highestFieldId": 23, + "highestFieldId": 25, "partitionKeys": [], "primaryKeys": [], "options": { @@ -155,5 +165,5 @@ "metadata.stats-mode": "none" }, "comment": null, - "timeMillis": 1789410866915 + "timeMillis": 1793168823492 } \ No newline at end of file diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-1 b/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-1 index 105c967eb..ff267abaf 100644 --- a/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-1 +++ b/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-1 @@ -2,14 +2,14 @@ "version": 3, "id": 1, "schemaId": 0, - "baseManifestList": "manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0", - "deltaManifestList": "manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1", + "baseManifestList": "manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0", + "deltaManifestList": "manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1", "totalRecordCount": 2, "deltaRecordCount": 2, - "commitUser": "020804d7-bcfa-4b8d-9c9c-176b700109e2", + "commitUser": "69ff4ed6-dff4-4e58-acb0-da04946ff985", "commitIdentifier": 9223372036854775807, "commitKind": "APPEND", - "timeMillis": 1789410866939, - "uuid": "5d3163c5-e36c-4e33-b690-fc8fb5f6438d", + "timeMillis": 1793168823524, + "uuid": "7a678363-830b-45f3-913a-8666e6c0ac02", "writerVersion": "python-2.1.dev-63ffd403648e1cf9848e32bb29a398c1b3b16ab1" } \ No newline at end of file diff --git a/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-2 b/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-2 index a641eacfe..9ea455676 100644 --- a/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-2 +++ b/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-2 @@ -2,14 +2,14 @@ "version": 3, "id": 2, "schemaId": 0, - "baseManifestList": "manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0", - "deltaManifestList": "manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1", + "baseManifestList": "manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0", + "deltaManifestList": "manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1", "totalRecordCount": 4, "deltaRecordCount": 2, - "commitUser": "c06f85ce-2b7e-449c-af6c-b7b5b0e50dce", + "commitUser": "e4715de3-7139-413b-91fe-ca187152a9dd", "commitIdentifier": 9223372036854775807, "commitKind": "APPEND", - "timeMillis": 1789410866974, - "uuid": "f7a0e42b-72a4-4785-8ec3-bbd59f1ea435", + "timeMillis": 1793168823541, + "uuid": "35aa33df-3390-4d54-95e3-8ef97e38f6ea", "writerVersion": "python-2.1.dev-63ffd403648e1cf9848e32bb29a398c1b3b16ab1" } \ No newline at end of file