From 26dd6ad063d69090a8cf933cc4426b644be2c1f7 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Sat, 10 Oct 2026 11:14:19 +0800 Subject: [PATCH 1/4] fix: continue aligned queries past all-null pages --- cpp/src/reader/aligned_chunk_reader.cc | 10 +- cpp/test/reader/aligned_null_page_test.cc | 233 ++++++++++++++++++ .../dataset/DataSetWithoutTimeGenerator.java | 35 ++- .../DataSetWithoutTimeGeneratorTest.java | 76 ++++++ 4 files changed, 333 insertions(+), 21 deletions(-) create mode 100644 cpp/test/reader/aligned_null_page_test.cc diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index 059570265..ad3f87a32 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -699,11 +699,15 @@ int AlignedChunkReader::decode_time_value_buf_into_tsblock( value_compressor_->after_uncompress(value_uncompressed_buf_); value_uncompressed_buf_ = nullptr; } - if (!prev_value_page_not_finish()) { - value_in_.reset(); - } if (!prev_time_page_not_finish()) { + // An aligned value page is complete once all its time rows have + // been consumed. All-null pages never decode their value stream, + // so a dictionary header or Gorilla terminator can remain unread. + value_in_.reset(); + value_decoder_->reset(); time_in_.reset(); + } else if (!prev_value_page_not_finish()) { + value_in_.reset(); } value_page_col_notnull_bitmap_.clear(); value_page_col_notnull_bitmap_.shrink_to_fit(); diff --git a/cpp/test/reader/aligned_null_page_test.cc b/cpp/test/reader/aligned_null_page_test.cc new file mode 100644 index 000000000..42c99eae7 --- /dev/null +++ b/cpp/test/reader/aligned_null_page_test.cc @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include "common/path.h" +#include "reader/qds_without_timegenerator.h" +#include "reader/tsfile_reader.h" +#include "writer/tsfile_writer.h" + +namespace storage { +namespace { + +using namespace common; + +// All-null value pages still contain codec headers (DICTIONARY) or a +// terminator (GORILLA). They must not prevent reading subsequent pages. +class AlignedNullPageReadTest + : public ::testing::TestWithParam> { + protected: + void SetUp() override { + ASSERT_EQ(libtsfile_init(), E_OK); + saved_page_rows_ = g_config_value_.page_writer_max_point_num_; + saved_block_memory_ = g_config_value_.tsblock_max_memory_; + g_config_value_.page_writer_max_point_num_ = std::get<1>(GetParam()); + // The larger pages span multiple result blocks. + g_config_value_.tsblock_max_memory_ = 4096; + file_ = "aligned_null_page_" + + std::to_string(static_cast(std::get<0>(GetParam()))) + + "_" + std::to_string(std::get<1>(GetParam())) + "_" + + std::to_string(std::get<2>(GetParam())) + ".tsfile"; + std::remove(file_.c_str()); + } + + void TearDown() override { + std::remove(file_.c_str()); + g_config_value_.page_writer_max_point_num_ = saved_page_rows_; + g_config_value_.tsblock_max_memory_ = saved_block_memory_; + libtsfile_destroy(); + } + + bool is_null(int row) const { + int page = row / std::get<1>(GetParam()); + return page == 0 || page == 2 || page == 3 || page == 6; + } + + void add_value(TsRecord& record, int row) { + switch (std::get<0>(GetParam())) { + case INT32: + record.add_point("value", static_cast(1000 + row)); + break; + case INT64: + record.add_point("value", static_cast(1000 + row)); + break; + case FLOAT: + record.add_point("value", static_cast(1000.25 + row)); + break; + case DOUBLE: + record.add_point("value", 1000.25 + row); + break; + case STRING: + record.add_point("value", + String("value-" + std::to_string(row))); + break; + default: + FAIL() << "Unexpected type"; + } + } + + void add_value(Tablet& tablet, int index, int row) { + switch (std::get<0>(GetParam())) { + case INT32: + ASSERT_EQ(tablet.add_value(index, 1, + static_cast(1000 + row)), + E_OK); + break; + case INT64: + ASSERT_EQ(tablet.add_value(index, 1, + static_cast(1000 + row)), + E_OK); + break; + case FLOAT: + ASSERT_EQ(tablet.add_value(index, 1, + static_cast(1000.25 + row)), + E_OK); + break; + case DOUBLE: + ASSERT_EQ(tablet.add_value(index, 1, 1000.25 + row), E_OK); + break; + case STRING: { + std::string value = "value-" + std::to_string(row); + ASSERT_EQ(tablet.add_value(index, 1, value.c_str()), E_OK); + break; + } + default: + FAIL() << "Unexpected type"; + } + } + + void check_value(Field* field, int row) { + if (is_null(row)) { + EXPECT_EQ(field->type_, NULL_TYPE) << "row=" << row; + return; + } + ASSERT_EQ(field->type_, std::get<0>(GetParam())) << "row=" << row; + switch (field->type_) { + case INT32: + EXPECT_EQ(field->value_.ival_, 1000 + row); + break; + case INT64: + EXPECT_EQ(field->value_.lval_, 1000 + row); + break; + case FLOAT: + EXPECT_FLOAT_EQ(field->value_.fval_, 1000.25f + row); + break; + case DOUBLE: + EXPECT_DOUBLE_EQ(field->value_.dval_, 1000.25 + row); + break; + case STRING: + EXPECT_EQ(field->get_string_value()->to_std_string(), + "value-" + std::to_string(row)); + break; + default: + FAIL() << "Unexpected type"; + } + } + + std::string file_; + uint32_t saved_page_rows_ = 0; + uint32_t saved_block_memory_ = 0; +}; + +TEST_P(AlignedNullPageReadTest, + ReadsValuesAfterLeadingAndIntermediateNullPages) { + const TSDataType type = std::get<0>(GetParam()); + const TSEncoding encoding = type == STRING ? DICTIONARY : GORILLA; + const std::string device = "root.null_pages"; + const int rows = 7 * std::get<1>(GetParam()); + const int mode = std::get<2>(GetParam()); + { + TsFileWriter writer; + ASSERT_EQ(writer.open(file_), E_OK); + std::vector schemas{ + new MeasurementSchema("anchor", INT64, PLAIN, UNCOMPRESSED), + new MeasurementSchema("value", type, encoding, UNCOMPRESSED)}; + ASSERT_EQ(writer.register_aligned_timeseries(device, schemas), E_OK); + for (int row = 0; row < rows;) { + if (mode == 0 || (mode == 2 && row % 3 == 0)) { + TsRecord record(row, device); + record.add_point("anchor", static_cast(row)); + if (is_null(row)) + record.points_.emplace_back("value"); + else + add_value(record, row); + ASSERT_EQ(writer.write_record_aligned(record), E_OK); + ++row; + } else { + const int count = std::min(11, rows - row); + auto schema = + std::make_shared>(); + schema->emplace_back("anchor", INT64, PLAIN, UNCOMPRESSED); + schema->emplace_back("value", type, encoding, UNCOMPRESSED); + Tablet tablet(device, schema, count); + ASSERT_EQ(tablet.err_code_, E_OK); + for (int index = 0; index < count; ++index) { + ASSERT_EQ(tablet.add_timestamp(index, row + index), E_OK); + ASSERT_EQ(tablet.add_value( + index, 0, static_cast(row + index)), + E_OK); + if (!is_null(row + index)) + add_value(tablet, index, row + index); + } + ASSERT_EQ(writer.write_tablet_aligned(tablet), E_OK); + row += count; + } + } + ASSERT_EQ(writer.flush(), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + TsFileReader reader; + ASSERT_EQ(reader.open(file_), E_OK); + ResultSet* result = nullptr; + std::vector paths{Path(device + ".anchor"), Path(device + ".value")}; + ASSERT_EQ(reader.query(QueryExpression::create(paths, nullptr), result), + E_OK); + int row = 0; + bool more = false; + while (true) { + ASSERT_EQ(result->next(more), E_OK); + if (!more) break; + auto* record = result->get_row_record(); + ASSERT_LT(row, rows); + EXPECT_EQ(record->get_timestamp(), row); + EXPECT_EQ(record->get_field(1)->value_.lval_, row); + check_value(record->get_field(2), row); + ++row; + } + EXPECT_EQ(row, rows); + reader.destroy_query_data_set(result); + ASSERT_EQ(reader.close(), E_OK); +} + +INSTANTIATE_TEST_SUITE_P( + CodecsAndPageBoundaries, AlignedNullPageReadTest, + ::testing::Combine(::testing::Values(INT32, INT64, FLOAT, DOUBLE, STRING), + ::testing::Values(1u, 7u, 256u), + ::testing::Values(0, 1, 2))); + +} // namespace +} // namespace storage diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java index 00faed8fd..1b6b525bb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java @@ -71,21 +71,25 @@ private void initHeap() throws IOException { timeSet = new LongOpenHashSet(seriesCount); for (int i = 0; i < paths.size(); i++) { - AbstractFileSeriesReader reader = readers.get(i); - if (!reader.hasNextBatch()) { - batchDataList.add(new BatchData()); - hasDataRemaining.add(false); - } else { - batchDataList.add(reader.nextBatch()); - hasDataRemaining.add(true); + BatchData data = nextNonEmptyBatch(readers.get(i)); + batchDataList.add(data == null ? new BatchData() : data); + hasDataRemaining.add(data != null); + if (data != null) { + timeHeapPut(data.currentTime()); } } + } - for (BatchData data : batchDataList) { + private static BatchData nextNonEmptyBatch(AbstractFileSeriesReader reader) throws IOException { + // An aligned page or a filtered batch can contain no selected rows while + // subsequent pages still contain values. + while (reader.hasNextBatch()) { + BatchData data = reader.nextBatch(); if (data.hasCurrent()) { - timeHeapPut(data.currentTime()); + return data; } } + return null; } @Override @@ -113,15 +117,10 @@ public RowRecord nextWithoutConstraint() throws IOException { data.next(); if (!data.hasCurrent()) { - AbstractFileSeriesReader reader = readers.get(i); - if (reader.hasNextBatch()) { - data = reader.nextBatch(); - if (data.hasCurrent()) { - batchDataList.set(i, data); - timeHeapPut(data.currentTime()); - } else { - hasDataRemaining.set(i, false); - } + data = nextNonEmptyBatch(readers.get(i)); + if (data != null) { + batchDataList.set(i, data); + timeHeapPut(data.currentTime()); } else { hasDataRemaining.set(i, false); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGeneratorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGeneratorTest.java index d027f75bd..c9cb818db 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGeneratorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGeneratorTest.java @@ -101,6 +101,82 @@ public void testMultiWayMergeAcrossBatches() throws IOException { assertEquals(Arrays.asList(1L, 2L, 3L), times); } + @Test + public void testSkipsLeadingAndIntermediateEmptyBatches() throws IOException { + BatchData first = batchOf(TSDataType.INT64, new long[] {1}, new long[] {10}); + BatchData last = batchOf(TSDataType.INT64, new long[] {4}, new long[] {40}); + DataSetWithoutTimeGenerator dataSet = + new DataSetWithoutTimeGenerator( + Collections.singletonList(new Path("root.d1", "s0", true)), + Collections.singletonList(TSDataType.INT64), + Collections.singletonList( + new FakeSeriesReader( + new BatchData(), + new BatchData(), + first, + new BatchData(), + new BatchData(), + last, + new BatchData()))); + + assertTrue(dataSet.hasNext()); + RowRecord firstRow = dataSet.next(); + assertEquals(1L, firstRow.getTimestamp()); + assertEquals(10L, firstRow.getFields().get(0).getLongV()); + assertTrue(dataSet.hasNext()); + RowRecord lastRow = dataSet.next(); + assertEquals(4L, lastRow.getTimestamp()); + assertEquals(40L, lastRow.getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + + @Test + public void testEmptyBatchesDoNotHideSparseSeriesDuringMerge() throws IOException { + BatchData anchor = + batchOf(TSDataType.INT64, new long[] {0, 1, 2, 3, 4, 5}, new long[] {0, 1, 2, 3, 4, 5}); + BatchData first = batchOf(TSDataType.INT64, new long[] {1}, new long[] {101}); + BatchData last = batchOf(TSDataType.INT64, new long[] {4}, new long[] {104}); + DataSetWithoutTimeGenerator dataSet = + new DataSetWithoutTimeGenerator( + Arrays.asList(new Path("root.d1", "s0", true), new Path("root.d1", "s1", true)), + Arrays.asList(TSDataType.INT64, TSDataType.INT64), + Arrays.asList( + new FakeSeriesReader(anchor), + new FakeSeriesReader( + new BatchData(), + first, + new BatchData(), + new BatchData(), + last, + new BatchData(), + new BatchData()))); + + for (int time = 0; time < 6; time++) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + assertEquals(time, row.getTimestamp()); + assertEquals(time, row.getFields().get(0).getLongV()); + if (time == 1 || time == 4) { + assertEquals(100L + time, row.getFields().get(1).getLongV()); + } else { + assertNull(row.getFields().get(1)); + } + } + assertFalse(dataSet.hasNext()); + } + + @Test + public void testOnlyEmptyBatchesAreExhausted() throws IOException { + DataSetWithoutTimeGenerator dataSet = + new DataSetWithoutTimeGenerator( + Collections.singletonList(new Path("root.d1", "s0", true)), + Collections.singletonList(TSDataType.INT64), + Collections.singletonList( + new FakeSeriesReader(new BatchData(), new BatchData(), new BatchData()))); + + assertFalse(dataSet.hasNext()); + } + private static BatchData batchOf(TSDataType type, long[] times, long[] values) { BatchData batchData = new BatchData(type); for (int i = 0; i < times.length; i++) { From 2545ea7a4be6164dc0bf191420ce04d17f05c6e1 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Sat, 10 Oct 2026 11:33:43 +0800 Subject: [PATCH 2/4] fix(cpp): write canonical all-null pages and revert Java changes --- cpp/src/file/restorable_tsfile_io_writer.cc | 8 ++ cpp/src/writer/value_chunk_writer.cc | 4 + cpp/src/writer/value_page_writer.cc | 24 ++++-- cpp/test/reader/aligned_null_page_test.cc | 76 +++++++++++++++++-- cpp/test/writer/value_chunk_writer_test.cc | 63 ++++++++++++++- cpp/test/writer/value_page_writer_test.cc | 33 ++++++++ .../dataset/DataSetWithoutTimeGenerator.java | 35 ++++----- .../DataSetWithoutTimeGeneratorTest.java | 76 ------------------- 8 files changed, 210 insertions(+), 109 deletions(-) diff --git a/cpp/src/file/restorable_tsfile_io_writer.cc b/cpp/src/file/restorable_tsfile_io_writer.cc index bd9824a65..9846f1ff9 100644 --- a/cpp/src/file/restorable_tsfile_io_writer.cc +++ b/cpp/src/file/restorable_tsfile_io_writer.cc @@ -54,6 +54,7 @@ namespace { const int HEADER_LEN = MAGIC_STRING_TSFILE_LEN + 1; // magic + version const int BUF_SIZE = 4096; const unsigned char kTimeChunkTypeMask = 0x80; +const unsigned char kValueChunkTypeMask = 0x40; // ----------------------------------------------------------------------------- // Self-check helpers: read file, parse chunk header, recover chunk statistics @@ -263,6 +264,13 @@ static int recover_chunk_statistic( kTimeChunkTypeMask) != 0; PageHeader ph; int ret = ph.deserialize_from(bs, false, chdr.data_type_); + if (ret == common::E_OK && ph.uncompressed_size_ == 0 && + (static_cast(chdr.chunk_type_) & kValueChunkTypeMask) != + 0) { + // A single all-null aligned value page has no payload or statistics. + return bs.remaining_size() == 0 ? common::E_OK + : common::E_TSFILE_CORRUPTED; + } if (ret != common::E_OK || ph.compressed_size_ == 0 || bs.remaining_size() < ph.compressed_size_) { // Align with Java selfCheck behavior: malformed/incomplete page in this diff --git a/cpp/src/writer/value_chunk_writer.cc b/cpp/src/writer/value_chunk_writer.cc index 88d0ea0f1..182471c71 100644 --- a/cpp/src/writer/value_chunk_writer.cc +++ b/cpp/src/writer/value_chunk_writer.cc @@ -159,6 +159,10 @@ void ValueChunkWriter::save_first_page_data( int ValueChunkWriter::write_first_page_data(ByteStream& pages_data, bool with_statistic) { int ret = E_OK; + if (first_page_data_.uncompressed_size_ == 0) { + // The empty first page is already complete in chunk_data_. + return ret; + } if (with_statistic && RET_FAIL(first_page_statistic_->serialize_to(pages_data))) { } else if (RET_FAIL( diff --git a/cpp/src/writer/value_page_writer.cc b/cpp/src/writer/value_page_writer.cc index c538ea2fa..7e9b09db7 100644 --- a/cpp/src/writer/value_page_writer.cc +++ b/cpp/src/writer/value_page_writer.cc @@ -151,11 +151,16 @@ int ValuePageWriter::write_to_chunk(ByteStream& pages_data, bool write_header, << pages_data.total_size() << " of chunk_data." << std::endl; #endif int ret = E_OK; - if (RET_FAIL(prepare_end_page())) { - return ret; - } - if (RET_FAIL(cur_page_data_.init(col_notnull_bitmap_out_stream_, - value_out_stream_, compressor_, size_))) { + if (statistic_->count_ == 0) { + // Match the aligned empty-page format: a single zero varint, with + // no compressed size, statistics, bitmap, or codec payload. + cur_page_data_ = ValuePageData(); + } else { + if (RET_FAIL(prepare_end_page())) { + return ret; + } + ret = cur_page_data_.init(col_notnull_bitmap_out_stream_, + value_out_stream_, compressor_, size_); } col_notnull_bitmap_.clear(); size_ = 0; @@ -164,13 +169,15 @@ int ValuePageWriter::write_to_chunk(ByteStream& pages_data, bool write_header, if (IS_SUCC(ret) && write_header) { if (RET_FAIL(SerializationUtil::write_var_uint( cur_page_data_.uncompressed_size_, pages_data))) { - } else if (RET_FAIL(SerializationUtil::write_var_uint( + } else if (cur_page_data_.uncompressed_size_ != 0 && + RET_FAIL(SerializationUtil::write_var_uint( cur_page_data_.compressed_size_, pages_data))) { } } // std::cout << "ValuePageWriter::write_to_chunk after write_header. pos=" // << pages_data.total_size() << std::endl; - if (IS_SUCC(ret) && write_statistic) { + if (IS_SUCC(ret) && write_statistic && + cur_page_data_.uncompressed_size_ != 0) { if (RET_FAIL(statistic_->serialize_to(pages_data))) { } } @@ -178,7 +185,8 @@ int ValuePageWriter::write_to_chunk(ByteStream& pages_data, bool write_header, // pages_data.total_size() << std::endl; DEBUG_print_byte_stream("In // ValuePageWriter::write_to_chunk, before writer page data: pages_data = ", // pages_data); - if (IS_SUCC(ret) && write_data_to_chunk_data) { + if (IS_SUCC(ret) && write_data_to_chunk_data && + cur_page_data_.compressed_size_ != 0) { // DEBUG_hex_dump_buf("cur_page_data_.compressed_buf_ = ", // cur_page_data_.compressed_buf_, cur_page_data_.compressed_size_); if (RET_FAIL(pages_data.write_buf(cur_page_data_.compressed_buf_, diff --git a/cpp/test/reader/aligned_null_page_test.cc b/cpp/test/reader/aligned_null_page_test.cc index 42c99eae7..7d3662b3f 100644 --- a/cpp/test/reader/aligned_null_page_test.cc +++ b/cpp/test/reader/aligned_null_page_test.cc @@ -21,11 +21,13 @@ #include #include +#include #include #include #include #include "common/path.h" +#include "encoding/encoder_factory.h" #include "reader/qds_without_timegenerator.h" #include "reader/tsfile_reader.h" #include "writer/tsfile_writer.h" @@ -35,10 +37,63 @@ namespace { using namespace common; -// All-null value pages still contain codec headers (DICTIONARY) or a -// terminator (GORILLA). They must not prevent reading subsequent pages. +// Recreate the older C++ representation so reader compatibility is tested +// independently of the corrected empty-page writer. +void use_legacy_null_pages(ValueChunkWriter* writer, TSDataType type, + TSEncoding encoding, uint32_t page_rows) { + ByteStream rewritten(1024, MOD_DEFAULT); + ByteStream& data = writer->get_chunk_data(); + std::unique_ptr encoder( + EncoderFactory::alloc_value_encoder(encoding, type), + EncoderFactory::free); + std::unique_ptr empty_stat( + StatisticFactory::alloc_statistic(type), StatisticFactory::free); + ASSERT_NE(encoder, nullptr); + ASSERT_NE(empty_stat, nullptr); + for (int page = 0; page < writer->num_of_pages(); ++page) { + PageHeader header; + ASSERT_EQ(header.deserialize_from(data, true, type), E_OK); + if (header.uncompressed_size_ == 0) { + ByteStream payload(1024, MOD_DEFAULT); + encoder->reset(); + ASSERT_EQ(encoder->flush(payload), E_OK); + std::vector bitmap((page_rows + 7) / 8, 0); + const uint32_t size = + sizeof(uint32_t) + bitmap.size() + payload.total_size(); + ASSERT_EQ(SerializationUtil::write_var_uint(size, rewritten), E_OK); + ASSERT_EQ(SerializationUtil::write_var_uint(size, rewritten), E_OK); + ASSERT_EQ(empty_stat->serialize_to(rewritten), E_OK); + ASSERT_EQ(SerializationUtil::write_ui32(page_rows, rewritten), + E_OK); + ASSERT_EQ(rewritten.write_buf(bitmap.data(), bitmap.size()), E_OK); + ASSERT_EQ(merge_byte_stream(rewritten, payload), E_OK); + } else { + ASSERT_EQ(SerializationUtil::write_var_uint( + header.uncompressed_size_, rewritten), + E_OK); + ASSERT_EQ(SerializationUtil::write_var_uint(header.compressed_size_, + rewritten), + E_OK); + ASSERT_EQ(header.statistic_->serialize_to(rewritten), E_OK); + std::vector payload(header.compressed_size_); + uint32_t read = 0; + ASSERT_EQ(data.read_buf(payload.data(), payload.size(), read), + E_OK); + ASSERT_EQ(read, payload.size()); + ASSERT_EQ(rewritten.write_buf(payload.data(), payload.size()), + E_OK); + } + } + ASSERT_EQ(data.remaining_size(), 0u); + data.reset(); + ASSERT_EQ(merge_byte_stream(data, rewritten), E_OK); +} + +// Leading and intermediate all-null value pages must not prevent reading +// subsequent pages, regardless of the value codec or write API. class AlignedNullPageReadTest - : public ::testing::TestWithParam> { + : public ::testing::TestWithParam< + std::tuple> { protected: void SetUp() override { ASSERT_EQ(libtsfile_init(), E_OK); @@ -50,7 +105,8 @@ class AlignedNullPageReadTest file_ = "aligned_null_page_" + std::to_string(static_cast(std::get<0>(GetParam()))) + "_" + std::to_string(std::get<1>(GetParam())) + "_" + - std::to_string(std::get<2>(GetParam())) + ".tsfile"; + std::to_string(std::get<2>(GetParam())) + "_" + + std::to_string(std::get<3>(GetParam())) + ".tsfile"; std::remove(file_.c_str()); } @@ -81,8 +137,8 @@ class AlignedNullPageReadTest record.add_point("value", 1000.25 + row); break; case STRING: - record.add_point("value", - String("value-" + std::to_string(row))); + record_string_ = "value-" + std::to_string(row); + record.add_point("value", String(record_string_)); break; default: FAIL() << "Unexpected type"; @@ -148,6 +204,7 @@ class AlignedNullPageReadTest } std::string file_; + std::string record_string_; uint32_t saved_page_rows_ = 0; uint32_t saved_block_memory_ = 0; }; @@ -196,6 +253,11 @@ TEST_P(AlignedNullPageReadTest, row += count; } } + if (std::get<3>(GetParam())) { + ASSERT_NO_FATAL_FAILURE( + use_legacy_null_pages(schemas[1]->value_chunk_writer_, type, + encoding, std::get<1>(GetParam()))); + } ASSERT_EQ(writer.flush(), E_OK); ASSERT_EQ(writer.close(), E_OK); } @@ -227,7 +289,7 @@ INSTANTIATE_TEST_SUITE_P( CodecsAndPageBoundaries, AlignedNullPageReadTest, ::testing::Combine(::testing::Values(INT32, INT64, FLOAT, DOUBLE, STRING), ::testing::Values(1u, 7u, 256u), - ::testing::Values(0, 1, 2))); + ::testing::Values(0, 1, 2), ::testing::Bool())); } // namespace } // namespace storage diff --git a/cpp/test/writer/value_chunk_writer_test.cc b/cpp/test/writer/value_chunk_writer_test.cc index c7250e4d5..fe1e76cf2 100644 --- a/cpp/test/writer/value_chunk_writer_test.cc +++ b/cpp/test/writer/value_chunk_writer_test.cc @@ -20,6 +20,8 @@ #include +#include + #include "common/allocator/byte_stream.h" #include "common/config/config.h" #include "common/statistic.h" @@ -94,4 +96,63 @@ TEST_F(ValueChunkWriterTest, DestroyChunkWriter) { value_chunk_writer.destroy(); EXPECT_EQ(value_chunk_writer.get_chunk_statistic(), nullptr); EXPECT_EQ(value_chunk_writer.get_chunk_data().total_size(), 0); -} \ No newline at end of file +} + +TEST_F(ValueChunkWriterTest, EmptyPagesDoNotHaveStatisticsOrData) { + const std::vector> patterns{ + {true}, + {true, true}, + {true, false}, + {false, true}, + {true, true, false, true, false, true}}; + for (const auto& pattern : patterns) { + for (bool seal_last_page : {false, true}) { + SCOPED_TRACE(::testing::Message() + << "pages=" << pattern.size() + << " seal_last_page=" << seal_last_page); + ValueChunkWriter writer; + ASSERT_EQ(writer.init("value", DOUBLE, GORILLA, UNCOMPRESSED), + E_OK); + writer.set_enable_page_seal_if_full(false); + int64_t timestamp = 0; + for (size_t page = 0; page < pattern.size(); ++page) { + for (int row = 0; row < 9; ++row) { + ASSERT_EQ(writer.write(timestamp++, 42.0, pattern[page]), + E_OK); + } + if (page + 1 < pattern.size() || seal_last_page) { + ASSERT_EQ(writer.seal_current_page(), E_OK); + } + } + ASSERT_EQ(writer.end_encode_chunk(), E_OK); + ASSERT_EQ(writer.num_of_pages(), pattern.size()); + ByteStream& data = writer.get_chunk_data(); + for (bool all_null : pattern) { + const auto start = data.read_pos(); + PageHeader header; + ASSERT_EQ( + header.deserialize_from(data, pattern.size() > 1, DOUBLE), + E_OK); + if (all_null) { + EXPECT_EQ(data.read_pos() - start, 1u); + EXPECT_EQ(header.uncompressed_size_, 0u); + EXPECT_EQ(header.compressed_size_, 0u); + EXPECT_EQ(header.statistic_, nullptr); + } else { + ASSERT_GT(header.compressed_size_, 0u); + if (pattern.size() > 1) { + ASSERT_NE(header.statistic_, nullptr); + EXPECT_EQ(header.statistic_->count_, 9u); + } + std::vector payload(header.compressed_size_); + uint32_t read = 0; + ASSERT_EQ( + data.read_buf(payload.data(), payload.size(), read), + E_OK); + ASSERT_EQ(read, payload.size()); + } + } + EXPECT_EQ(data.remaining_size(), 0u); + } + } +} diff --git a/cpp/test/writer/value_page_writer_test.cc b/cpp/test/writer/value_page_writer_test.cc index be04586ee..950b278de 100644 --- a/cpp/test/writer/value_page_writer_test.cc +++ b/cpp/test/writer/value_page_writer_test.cc @@ -139,3 +139,36 @@ TEST_F(ValuePageWriterTest, WriteBatchCommitsStateAfterEncode) { ASSERT_NE(stat, nullptr); EXPECT_EQ(stat->count_, 3u); } + +TEST_F(ValuePageWriterTest, AllNullPagesUseEmptyPageMarker) { + for (auto type : {TSDataType::INT64, TSDataType::FLOAT, TSDataType::DOUBLE, + TSDataType::STRING}) { + for (auto encoding : {TSEncoding::PLAIN, type == TSDataType::STRING + ? TSEncoding::DICTIONARY + : TSEncoding::GORILLA}) { + for (bool write_statistic : {false, true}) { + SCOPED_TRACE(::testing::Message() + << "type=" << static_cast(type) + << " encoding=" << static_cast(encoding) + << " statistics=" << write_statistic); + ValuePageWriter writer; + ASSERT_EQ(writer.init(type, encoding, UNCOMPRESSED), E_OK); + ASSERT_EQ(writer.write_null_rows(9), E_OK); + ByteStream data(1024, MOD_DEFAULT); + ASSERT_EQ( + writer.write_to_chunk(data, true, write_statistic, true), + E_OK); + // The complete page is one zero varint, without compressed + // size, statistics, bitmap, or codec payload. + EXPECT_EQ(data.total_size(), 1u); + uint32_t uncompressed_size = 1; + ASSERT_EQ( + SerializationUtil::read_var_uint(uncompressed_size, data), + E_OK); + EXPECT_EQ(uncompressed_size, 0u); + EXPECT_EQ(data.remaining_size(), 0u); + writer.destroy_page_data(); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java index 1b6b525bb..00faed8fd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java @@ -71,25 +71,21 @@ private void initHeap() throws IOException { timeSet = new LongOpenHashSet(seriesCount); for (int i = 0; i < paths.size(); i++) { - BatchData data = nextNonEmptyBatch(readers.get(i)); - batchDataList.add(data == null ? new BatchData() : data); - hasDataRemaining.add(data != null); - if (data != null) { - timeHeapPut(data.currentTime()); + AbstractFileSeriesReader reader = readers.get(i); + if (!reader.hasNextBatch()) { + batchDataList.add(new BatchData()); + hasDataRemaining.add(false); + } else { + batchDataList.add(reader.nextBatch()); + hasDataRemaining.add(true); } } - } - private static BatchData nextNonEmptyBatch(AbstractFileSeriesReader reader) throws IOException { - // An aligned page or a filtered batch can contain no selected rows while - // subsequent pages still contain values. - while (reader.hasNextBatch()) { - BatchData data = reader.nextBatch(); + for (BatchData data : batchDataList) { if (data.hasCurrent()) { - return data; + timeHeapPut(data.currentTime()); } } - return null; } @Override @@ -117,10 +113,15 @@ public RowRecord nextWithoutConstraint() throws IOException { data.next(); if (!data.hasCurrent()) { - data = nextNonEmptyBatch(readers.get(i)); - if (data != null) { - batchDataList.set(i, data); - timeHeapPut(data.currentTime()); + AbstractFileSeriesReader reader = readers.get(i); + if (reader.hasNextBatch()) { + data = reader.nextBatch(); + if (data.hasCurrent()) { + batchDataList.set(i, data); + timeHeapPut(data.currentTime()); + } else { + hasDataRemaining.set(i, false); + } } else { hasDataRemaining.set(i, false); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGeneratorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGeneratorTest.java index c9cb818db..d027f75bd 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGeneratorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGeneratorTest.java @@ -101,82 +101,6 @@ public void testMultiWayMergeAcrossBatches() throws IOException { assertEquals(Arrays.asList(1L, 2L, 3L), times); } - @Test - public void testSkipsLeadingAndIntermediateEmptyBatches() throws IOException { - BatchData first = batchOf(TSDataType.INT64, new long[] {1}, new long[] {10}); - BatchData last = batchOf(TSDataType.INT64, new long[] {4}, new long[] {40}); - DataSetWithoutTimeGenerator dataSet = - new DataSetWithoutTimeGenerator( - Collections.singletonList(new Path("root.d1", "s0", true)), - Collections.singletonList(TSDataType.INT64), - Collections.singletonList( - new FakeSeriesReader( - new BatchData(), - new BatchData(), - first, - new BatchData(), - new BatchData(), - last, - new BatchData()))); - - assertTrue(dataSet.hasNext()); - RowRecord firstRow = dataSet.next(); - assertEquals(1L, firstRow.getTimestamp()); - assertEquals(10L, firstRow.getFields().get(0).getLongV()); - assertTrue(dataSet.hasNext()); - RowRecord lastRow = dataSet.next(); - assertEquals(4L, lastRow.getTimestamp()); - assertEquals(40L, lastRow.getFields().get(0).getLongV()); - assertFalse(dataSet.hasNext()); - } - - @Test - public void testEmptyBatchesDoNotHideSparseSeriesDuringMerge() throws IOException { - BatchData anchor = - batchOf(TSDataType.INT64, new long[] {0, 1, 2, 3, 4, 5}, new long[] {0, 1, 2, 3, 4, 5}); - BatchData first = batchOf(TSDataType.INT64, new long[] {1}, new long[] {101}); - BatchData last = batchOf(TSDataType.INT64, new long[] {4}, new long[] {104}); - DataSetWithoutTimeGenerator dataSet = - new DataSetWithoutTimeGenerator( - Arrays.asList(new Path("root.d1", "s0", true), new Path("root.d1", "s1", true)), - Arrays.asList(TSDataType.INT64, TSDataType.INT64), - Arrays.asList( - new FakeSeriesReader(anchor), - new FakeSeriesReader( - new BatchData(), - first, - new BatchData(), - new BatchData(), - last, - new BatchData(), - new BatchData()))); - - for (int time = 0; time < 6; time++) { - assertTrue(dataSet.hasNext()); - RowRecord row = dataSet.next(); - assertEquals(time, row.getTimestamp()); - assertEquals(time, row.getFields().get(0).getLongV()); - if (time == 1 || time == 4) { - assertEquals(100L + time, row.getFields().get(1).getLongV()); - } else { - assertNull(row.getFields().get(1)); - } - } - assertFalse(dataSet.hasNext()); - } - - @Test - public void testOnlyEmptyBatchesAreExhausted() throws IOException { - DataSetWithoutTimeGenerator dataSet = - new DataSetWithoutTimeGenerator( - Collections.singletonList(new Path("root.d1", "s0", true)), - Collections.singletonList(TSDataType.INT64), - Collections.singletonList( - new FakeSeriesReader(new BatchData(), new BatchData(), new BatchData()))); - - assertFalse(dataSet.hasNext()); - } - private static BatchData batchOf(TSDataType type, long[] times, long[] values) { BatchData batchData = new BatchData(type); for (int i = 0; i < times.length; i++) { From 833bbb117307dd101547bebfe204979e1041d02c Mon Sep 17 00:00:00 2001 From: ColinLee Date: Sat, 10 Oct 2026 12:14:37 +0800 Subject: [PATCH 3/4] test(cpp): refresh CLI goldens for compact all-null pages --- .../tools/golden/empty_field_meta_csv.txt | 2 +- .../tools/golden/empty_field_meta_ndjson.txt | 2 +- .../tools/golden/empty_field_meta_table.txt | 2 +- cpp/test/tools/golden/empty_field_sketch.txt | 62 +++++++++---------- 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/cpp/test/tools/golden/empty_field_meta_csv.txt b/cpp/test/tools/golden/empty_field_meta_csv.txt index efef40032..58b0ab0bf 100644 --- a/cpp/test/tools/golden/empty_field_meta_csv.txt +++ b/cpp/test/tools/golden/empty_field_meta_csv.txt @@ -1,2 +1,2 @@ size_bytes,format_version,model -457,4,table +451,4,table diff --git a/cpp/test/tools/golden/empty_field_meta_ndjson.txt b/cpp/test/tools/golden/empty_field_meta_ndjson.txt index 5be88b629..d339bc6c9 100644 --- a/cpp/test/tools/golden/empty_field_meta_ndjson.txt +++ b/cpp/test/tools/golden/empty_field_meta_ndjson.txt @@ -1 +1 @@ -{"size_bytes":"457","format_version":"4","model":"table"} +{"size_bytes":"451","format_version":"4","model":"table"} diff --git a/cpp/test/tools/golden/empty_field_meta_table.txt b/cpp/test/tools/golden/empty_field_meta_table.txt index deb13ed43..380edfebf 100644 --- a/cpp/test/tools/golden/empty_field_meta_table.txt +++ b/cpp/test/tools/golden/empty_field_meta_table.txt @@ -1,2 +1,2 @@ size_bytes format_version model -457 4 table +451 4 table diff --git a/cpp/test/tools/golden/empty_field_sketch.txt b/cpp/test/tools/golden/empty_field_sketch.txt index 4ab4551ac..9a5095f41 100644 --- a/cpp/test/tools/golden/empty_field_sketch.txt +++ b/cpp/test/tools/golden/empty_field_sketch.txt @@ -1,6 +1,6 @@ -------------------------------- TsFile Sketch -------------------------------- file path: tsfile_cli_empty_field_golden.tsfile -file length: 457 +file length: 451 POSITION| CONTENT -------- ------- @@ -15,41 +15,41 @@ file length: 457 34| [Page Header] HeaderSize:2, UncompressedSize:24, CompressedSize:19 36| [Page Data] Size:19 55| [Chunk] of sensors.beijing.r1.temp, startTime: 0 endTime: 0 count: 0 [minValue:0.0,maxValue:0.0,firstValue:0.0,lastValue:0.0,sumValue:0.0] - 55| [Chunk Header] marker=69, measurementID=temp, dataSize=7, dataType=DOUBLE, compressionType=UNCOMPRESSED, encodingType=PLAIN, size=10 - 65| [Page Header] HeaderSize:2, UncompressedSize:5, CompressedSize:5 - 67| [Page Data] Size:5 + 55| [Chunk Header] marker=69, measurementID=temp, dataSize=1, dataType=DOUBLE, compressionType=UNCOMPRESSED, encodingType=PLAIN, size=10 + 65| [Page Header] HeaderSize:1, UncompressedSize:0, CompressedSize:0 + 66| [Page Data] Size:0 ||||||||||||||||||||| [ChunkGroup] of sensors.beijing.r1 ends - 89| [marker] 2 - 90| [TimeseriesMetadata] of sensors.beijing.r1., tsDataType:VECTOR, sizeWithoutChunkMetadata:21, startTime: 1000 endTime: 2000 count: 2 - 111| [ChunkMetadata] offset=28, size=8 - 119| [TimeseriesMetadata] of sensors.beijing.r1.temp, tsDataType:DOUBLE, sizeWithoutChunkMetadata:65, startTime: 0 endTime: 0 count: 0 [minValue:0.0,maxValue:0.0,firstValue:0.0,lastValue:0.0,sumValue:0.0] - 184| [ChunkMetadata] offset=55, size=8 - 192| [MetadataIndexNode] - 192| childrenCnt=1 - 193| <, 90> - 202| endOffset=192 - 210| nodeType=LEAF_MEASUREMENT + 83| [marker] 2 + 84| [TimeseriesMetadata] of sensors.beijing.r1., tsDataType:VECTOR, sizeWithoutChunkMetadata:21, startTime: 1000 endTime: 2000 count: 2 + 105| [ChunkMetadata] offset=28, size=8 + 113| [TimeseriesMetadata] of sensors.beijing.r1.temp, tsDataType:DOUBLE, sizeWithoutChunkMetadata:65, startTime: 0 endTime: 0 count: 0 [minValue:0.0,maxValue:0.0,firstValue:0.0,lastValue:0.0,sumValue:0.0] + 178| [ChunkMetadata] offset=55, size=8 + 186| [MetadataIndexNode] + 186| childrenCnt=1 + 187| <, 84> + 196| endOffset=186 + 204| nodeType=LEAF_MEASUREMENT ||||||||||||||||||||| [TsFileMetadata] begins - 211| TableIndexRootCnt=1 - 215| [Table Name] sensors, size=11 - 226| [MetadataIndexNode] - 226| childrenCnt=1 - 227| - 255| endOffset=211 - 263| nodeType=LEAF_DEVICE - 264| TableSchemaCnt=1 - 268| [TableSchema] TableSchema{tableName='sensors', columnSchemas=[[site,STRING,PLAIN,,UNCOMPRESSED], [room,STRING,PLAIN,,UNCOMPRESSED], [temp,DOUBLE,PLAIN,,UNCOMPRESSED]], columnTypes=[TAG, TAG, FIELD]}, size=58 - 337| [Meta Offset] 89 - 345| [Bloom Filter Size] bit vector byte array length=325 - 349| [Bloom Filter] , filterCapacity=256, hashFunctionSize=5 + 205| TableIndexRootCnt=1 + 209| [Table Name] sensors, size=11 + 220| [MetadataIndexNode] + 220| childrenCnt=1 + 221| + 249| endOffset=205 + 257| nodeType=LEAF_DEVICE + 258| TableSchemaCnt=1 + 262| [TableSchema] TableSchema{tableName='sensors', columnSchemas=[[site,STRING,PLAIN,,UNCOMPRESSED], [room,STRING,PLAIN,,UNCOMPRESSED], [temp,DOUBLE,PLAIN,,UNCOMPRESSED]], columnTypes=[TAG, TAG, FIELD]}, size=58 + 331| [Meta Offset] 83 + 339| [Bloom Filter Size] bit vector byte array length=325 + 343| [Bloom Filter] , filterCapacity=256, hashFunctionSize=5 ||||||||||||||||||||| [TsFileMetadata] ends - 447| [TsFileMetadataSize] 236 - 451| [magic tail] TsFile - 457| END of TsFile + 441| [TsFileMetadataSize] 236 + 445| [magic tail] TsFile + 451| END of TsFile ---------------------------- IndexOfTimerseriesIndex Tree ----------------------------- sensors [MetadataIndex:LEAF_DEVICE] - └──────[sensors.beijing.r1,192] + └──────[sensors.beijing.r1,186] [MetadataIndex:LEAF_MEASUREMENT] - └──────[,90] + └──────[,84] ---------------------------------- TsFile Sketch End ---------------------------------- From 028188e0da76aeb7276c7527784a5944b6957e14 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Sat, 10 Oct 2026 12:32:43 +0800 Subject: [PATCH 4/4] test(cpp): focus null-page coverage on canonical format --- cpp/src/reader/aligned_chunk_reader.cc | 6 +-- cpp/test/reader/aligned_null_page_test.cc | 66 ++--------------------- 2 files changed, 6 insertions(+), 66 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index ad3f87a32..ff4552cc8 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -700,9 +700,9 @@ int AlignedChunkReader::decode_time_value_buf_into_tsblock( value_uncompressed_buf_ = nullptr; } if (!prev_time_page_not_finish()) { - // An aligned value page is complete once all its time rows have - // been consumed. All-null pages never decode their value stream, - // so a dictionary header or Gorilla terminator can remain unread. + // The time page determines the aligned value page's end. Even + // after decoding every value, Gorilla can leave a padding byte + // unread, which must not prevent advancing to the next page. value_in_.reset(); value_decoder_->reset(); time_in_.reset(); diff --git a/cpp/test/reader/aligned_null_page_test.cc b/cpp/test/reader/aligned_null_page_test.cc index 7d3662b3f..305d0c7cd 100644 --- a/cpp/test/reader/aligned_null_page_test.cc +++ b/cpp/test/reader/aligned_null_page_test.cc @@ -27,7 +27,6 @@ #include #include "common/path.h" -#include "encoding/encoder_factory.h" #include "reader/qds_without_timegenerator.h" #include "reader/tsfile_reader.h" #include "writer/tsfile_writer.h" @@ -37,63 +36,10 @@ namespace { using namespace common; -// Recreate the older C++ representation so reader compatibility is tested -// independently of the corrected empty-page writer. -void use_legacy_null_pages(ValueChunkWriter* writer, TSDataType type, - TSEncoding encoding, uint32_t page_rows) { - ByteStream rewritten(1024, MOD_DEFAULT); - ByteStream& data = writer->get_chunk_data(); - std::unique_ptr encoder( - EncoderFactory::alloc_value_encoder(encoding, type), - EncoderFactory::free); - std::unique_ptr empty_stat( - StatisticFactory::alloc_statistic(type), StatisticFactory::free); - ASSERT_NE(encoder, nullptr); - ASSERT_NE(empty_stat, nullptr); - for (int page = 0; page < writer->num_of_pages(); ++page) { - PageHeader header; - ASSERT_EQ(header.deserialize_from(data, true, type), E_OK); - if (header.uncompressed_size_ == 0) { - ByteStream payload(1024, MOD_DEFAULT); - encoder->reset(); - ASSERT_EQ(encoder->flush(payload), E_OK); - std::vector bitmap((page_rows + 7) / 8, 0); - const uint32_t size = - sizeof(uint32_t) + bitmap.size() + payload.total_size(); - ASSERT_EQ(SerializationUtil::write_var_uint(size, rewritten), E_OK); - ASSERT_EQ(SerializationUtil::write_var_uint(size, rewritten), E_OK); - ASSERT_EQ(empty_stat->serialize_to(rewritten), E_OK); - ASSERT_EQ(SerializationUtil::write_ui32(page_rows, rewritten), - E_OK); - ASSERT_EQ(rewritten.write_buf(bitmap.data(), bitmap.size()), E_OK); - ASSERT_EQ(merge_byte_stream(rewritten, payload), E_OK); - } else { - ASSERT_EQ(SerializationUtil::write_var_uint( - header.uncompressed_size_, rewritten), - E_OK); - ASSERT_EQ(SerializationUtil::write_var_uint(header.compressed_size_, - rewritten), - E_OK); - ASSERT_EQ(header.statistic_->serialize_to(rewritten), E_OK); - std::vector payload(header.compressed_size_); - uint32_t read = 0; - ASSERT_EQ(data.read_buf(payload.data(), payload.size(), read), - E_OK); - ASSERT_EQ(read, payload.size()); - ASSERT_EQ(rewritten.write_buf(payload.data(), payload.size()), - E_OK); - } - } - ASSERT_EQ(data.remaining_size(), 0u); - data.reset(); - ASSERT_EQ(merge_byte_stream(data, rewritten), E_OK); -} - // Leading and intermediate all-null value pages must not prevent reading // subsequent pages, regardless of the value codec or write API. class AlignedNullPageReadTest - : public ::testing::TestWithParam< - std::tuple> { + : public ::testing::TestWithParam> { protected: void SetUp() override { ASSERT_EQ(libtsfile_init(), E_OK); @@ -105,8 +51,7 @@ class AlignedNullPageReadTest file_ = "aligned_null_page_" + std::to_string(static_cast(std::get<0>(GetParam()))) + "_" + std::to_string(std::get<1>(GetParam())) + "_" + - std::to_string(std::get<2>(GetParam())) + "_" + - std::to_string(std::get<3>(GetParam())) + ".tsfile"; + std::to_string(std::get<2>(GetParam())) + ".tsfile"; std::remove(file_.c_str()); } @@ -253,11 +198,6 @@ TEST_P(AlignedNullPageReadTest, row += count; } } - if (std::get<3>(GetParam())) { - ASSERT_NO_FATAL_FAILURE( - use_legacy_null_pages(schemas[1]->value_chunk_writer_, type, - encoding, std::get<1>(GetParam()))); - } ASSERT_EQ(writer.flush(), E_OK); ASSERT_EQ(writer.close(), E_OK); } @@ -289,7 +229,7 @@ INSTANTIATE_TEST_SUITE_P( CodecsAndPageBoundaries, AlignedNullPageReadTest, ::testing::Combine(::testing::Values(INT32, INT64, FLOAT, DOUBLE, STRING), ::testing::Values(1u, 7u, 256u), - ::testing::Values(0, 1, 2), ::testing::Bool())); + ::testing::Values(0, 1, 2))); } // namespace } // namespace storage