Skip to content

Commit 438fcf4

Browse files
zjw1111zjw1111
committed
refactor(global-index): simplify validation and clarify score tests
1 parent f9af40e commit 438fcf4

6 files changed

Lines changed: 101 additions & 129 deletions

File tree

‎src/paimon/common/global_index/complete_index_score_batch_reader_test.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,8 @@ TEST_F(CompleteIndexScoreBatchReaderTest, TestGlobalScoresFollowRowIds) {
172172
arrow::struct_(fields),
173173
R"([[null, 12, 102], [null, 99, null], [null, 10, 100], [null, 18, 108]])")
174174
.ValueOrDie();
175+
const std::unordered_map<int64_t, float> scores_by_row_id = {
176+
{100, 1.0f}, {101, 2.0f}, {102, 3.0f}, {108, 9.0f}, {109, 10.0f}};
175177
for (bool remove_row_id : {false, true}) {
176178
auto expected_fields = fields;
177179
if (remove_row_id) {
@@ -189,9 +191,7 @@ TEST_F(CompleteIndexScoreBatchReaderTest, TestGlobalScoresFollowRowIds) {
189191
src_array, src_array->type(), RoaringBitmap32::From({0, 2, 3}), batch_size);
190192
inner->EnableRandomizeBatchSize(false);
191193
auto reader = std::make_unique<CompleteIndexScoreBatchReader>(
192-
std::move(inner),
193-
std::unordered_map<int64_t, float>{
194-
{100, 1.0f}, {101, 2.0f}, {102, 3.0f}, {108, 9.0f}, {109, 10.0f}},
194+
std::move(inner), std::unordered_map<int64_t, float>(scores_by_row_id),
195195
remove_row_id, GetArrowPool(pool_));
196196
ASSERT_OK_AND_ASSIGN(auto result_array,
197197
ReadResultCollector::CollectResult(std::move(reader)));

‎src/paimon/common/reader/blob_fallback_batch_reader_test.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,10 +225,10 @@ TEST_F(BlobFallbackBatchReaderTest, TestRowTrackingFieldsPreserved) {
225225
}
226226
arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", true)};
227227
if (with_row_id) {
228-
fields.push_back(SpecialFields::RowId().field_);
228+
fields.push_back(SpecialFields::RowId().ArrowField());
229229
}
230230
if (with_seq_num) {
231-
fields.push_back(SpecialFields::SequenceNumber().field_);
231+
fields.push_back(SpecialFields::SequenceNumber().ArrowField());
232232
}
233233
auto struct_type = arrow::struct_(fields);
234234
auto schema = arrow::schema(fields);

‎src/paimon/core/global_index/indexed_split_impl.h‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,8 @@
1919

2020
#pragma once
2121

22-
#include <algorithm>
2322
#include <cstddef>
2423
#include <cstdint>
25-
#include <limits>
2624
#include <memory>
2725
#include <optional>
2826
#include <string>
@@ -100,22 +98,7 @@ class IndexedSplitImpl : public IndexedSplit {
10098
}
10199
if (!scores_.empty()) {
102100
size_t row_count = 0;
103-
auto sorted_ranges = row_ranges_;
104-
std::sort(sorted_ranges.begin(), sorted_ranges.end());
105-
std::optional<int64_t> previous_end;
106-
for (const auto& range : sorted_ranges) {
107-
if (range.from < 0 || range.from > range.to) {
108-
return Status::Invalid("Invalid row id range in scored indexed split.");
109-
}
110-
if (range.to == std::numeric_limits<int64_t>::max()) {
111-
return Status::Invalid(
112-
"Row id range upper bound must be less than INT64_MAX in scored indexed "
113-
"split.");
114-
}
115-
if (previous_end && range.from <= previous_end.value()) {
116-
return Status::Invalid("Duplicate row id in scored indexed split.");
117-
}
118-
previous_end = range.to;
101+
for (const auto& range : row_ranges_) {
119102
row_count += range.Count();
120103
}
121104
if (row_count != scores_.size()) {

‎src/paimon/core/global_index/indexed_split_test.cpp‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919

2020
#include <cmath>
2121
#include <cstdint>
22-
#include <limits>
2322
#include <memory>
2423
#include <string>
2524
#include <utility>
@@ -221,32 +220,6 @@ TEST(IndexedSplitTest, TestValidate) {
221220
ASSERT_NOK_WITH_MSG(split.Validate(),
222221
"Scores length does not match row ranges in indexed split.");
223222
}
224-
{
225-
IndexedSplitImpl split(data_split, {Range(10, 12), Range(12, 13)},
226-
{1.0f, 2.0f, 3.0f, 4.0f, 5.0f});
227-
ASSERT_NOK_WITH_MSG(split.Validate(), "Duplicate row id in scored indexed split.");
228-
}
229-
{
230-
IndexedSplitImpl split(data_split, {Range(30, 31), Range(10, 12)},
231-
{1.0f, 2.0f, 3.0f, 4.0f, 5.0f});
232-
ASSERT_OK(split.Validate());
233-
}
234-
{
235-
IndexedSplitImpl split(data_split, {Range(-1, 0)}, {1.0f, 2.0f});
236-
ASSERT_NOK_WITH_MSG(split.Validate(), "Invalid row id range in scored indexed split.");
237-
}
238-
{
239-
constexpr int64_t max_row_id = std::numeric_limits<int64_t>::max();
240-
IndexedSplitImpl split(data_split, {Range(max_row_id, max_row_id)}, {1.0f});
241-
ASSERT_NOK_WITH_MSG(split.Validate(),
242-
"Row id range upper bound must be less than INT64_MAX");
243-
IndexedSplitImpl range_split(data_split, {Range(max_row_id - 1, max_row_id)}, {1.0f, 2.0f});
244-
ASSERT_NOK_WITH_MSG(range_split.Validate(),
245-
"Row id range upper bound must be less than INT64_MAX");
246-
IndexedSplitImpl valid_split(data_split, {Range(max_row_id - 2, max_row_id - 1)},
247-
{1.0f, 2.0f});
248-
ASSERT_OK(valid_split.Validate());
249-
}
250223
{
251224
std::vector<Range> row_ranges = {};
252225
IndexedSplitImpl split(data_split, row_ranges);

‎test/inte/blob_table_inte_test.cpp‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -977,8 +977,8 @@ TEST_P(BlobTableInteTest, TestBasic) {
977977
// read with row tracking
978978
auto expected_row_tracking_array = std::dynamic_pointer_cast<arrow::StructArray>(
979979
arrow::ipc::internal::json::ArrayFromJSON(
980-
arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_,
981-
SpecialFields::RowId().field_, fields_[2]}),
980+
arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().ArrowField(),
981+
SpecialFields::RowId().ArrowField(), fields_[2]}),
982982
R"([
983983
["new_blob", 1, 2, 0, "c"]
984984
])")
@@ -1138,8 +1138,8 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) {
11381138
fields_[0],
11391139
fields_[1],
11401140
fields_[2],
1141-
SpecialFields::RowId().field_,
1142-
SpecialFields::SequenceNumber().field_,
1141+
SpecialFields::RowId().ArrowField(),
1142+
SpecialFields::SequenceNumber().ArrowField(),
11431143
}),
11441144
R"([
11451145
[1, "a", "b", 0, 1],
@@ -1206,7 +1206,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) {
12061206

12071207
auto expected_with_row_id = std::dynamic_pointer_cast<arrow::StructArray>(
12081208
arrow::ipc::internal::json::ArrayFromJSON(
1209-
arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}),
1209+
arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().ArrowField()}),
12101210
R"([
12111211
[1, "a", "new_blob_0", 0],
12121212
[2, "b", "new_blob_1", 1]
@@ -1300,7 +1300,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) {
13001300
// row ids still come from the data files and stay aligned with the fallback result
13011301
auto expected_with_row_id = std::dynamic_pointer_cast<arrow::StructArray>(
13021302
arrow::ipc::internal::json::ArrayFromJSON(
1303-
arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}),
1303+
arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().ArrowField()}),
13041304
R"([
13051305
[1, "a", null, 0],
13061306
[2, "b", "updated_b", 1],
@@ -1449,7 +1449,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithDeletionVectors)
14491449

14501450
auto expected_with_row_id = std::dynamic_pointer_cast<arrow::StructArray>(
14511451
arrow::ipc::internal::json::ArrayFromJSON(
1452-
arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}),
1452+
arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().ArrowField()}),
14531453
R"([
14541454
[1, "a", "update1_0", 0],
14551455
[3, "c", "blob_2", 2]
@@ -1977,8 +1977,8 @@ TEST_P(BlobTableInteTest, TestMultipleAppendsDifferentFirstRowIds) {
19771977
fields_[0],
19781978
fields_[1],
19791979
fields_[2],
1980-
SpecialFields::RowId().field_,
1981-
SpecialFields::SequenceNumber().field_,
1980+
SpecialFields::RowId().ArrowField(),
1981+
SpecialFields::SequenceNumber().ArrowField(),
19821982
}),
19831983
R"([
19841984
[1, "a", "b", 0, 1],
@@ -2125,8 +2125,8 @@ TEST_P(BlobTableInteTest, TestExternalPath) {
21252125
// read with row tracking
21262126
auto expected_row_tracking_array = std::dynamic_pointer_cast<arrow::StructArray>(
21272127
arrow::ipc::internal::json::ArrayFromJSON(
2128-
arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_,
2129-
SpecialFields::SequenceNumber().field_}),
2128+
arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().ArrowField(),
2129+
SpecialFields::SequenceNumber().ArrowField()}),
21302130
R"([
21312131
["a", 10, "b", 0, 2],
21322132
["c", 20, "d", 1, 2]
@@ -2241,8 +2241,9 @@ TEST_P(BlobTableInteTest, TestPartitionWithPredicate) {
22412241

22422242
auto expected_row_tracking_array = std::dynamic_pointer_cast<arrow::StructArray>(
22432243
arrow::ipc::internal::json::ArrayFromJSON(
2244-
arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_,
2245-
SpecialFields::SequenceNumber().field_}),
2244+
arrow::struct_({fields_[0], fields_[1], fields_[2],
2245+
SpecialFields::RowId().ArrowField(),
2246+
SpecialFields::SequenceNumber().ArrowField()}),
22462247
R"([
22472248
[11, "2024", "a", 0, 1],
22482249
[11, "2025", "b", 1, 1],
@@ -2382,8 +2383,8 @@ TEST_P(BlobTableInteTest, TestIOException) {
23822383
bool read_run_complete = false;
23832384
auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
23842385
arrow::ipc::internal::json::ArrayFromJSON(
2385-
arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_,
2386-
SpecialFields::RowId().field_, fields_[2]}),
2386+
arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().ArrowField(),
2387+
SpecialFields::RowId().ArrowField(), fields_[2]}),
23872388
R"([
23882389
["a", 100, 2, 0, "c"],
23892390
["aa", 200, 2, 1, "cc"],

0 commit comments

Comments
 (0)