From fe2e0785496480baeea0ed0691427ad58dad6290 Mon Sep 17 00:00:00 2001 From: TsukiokaKogane Date: Sun, 25 Oct 2026 17:33:51 +0800 Subject: [PATCH] [refactor](changelog) Use explicit column mappings for reads and writes ### What problem does this PR solve? Related PR: #67456, #66472 Problem Summary: Changelog execution inferred current/before relationships from column names and layout positions, coupling query projection to storage layout and leaving publish retries without a stable write-time relationship. Plan explicit slot pairs for reads and immutable source/current/before UID snapshots for writes. Register snapshots against actual transaction IDs, freeze them at precommit/commit, and retain them for replay and lagging-replica republish. Propagate the same snapshots through remote commit, pending publish persistence and Cloud transaction contexts. Resolve a schema-bound CID layout once per write/publish operation and keep query protocol parsing in Scanner. Validate mappings before visibility, preserve key-only historical lookup, and propagate pending-record save failures before queueing asynchronous work. ### Release note Changelog reads and writes use explicit column mappings. Changelog requests requiring mappings now reject missing or malformed mappings. Mixed-version adapters and rolling-upgrade behavior are outside this implementation scope. ### Check List (For Author) - Test: 85 targeted FE unit tests passed. FE and ASAN BE builds passed. FE Checkstyle, clang-format 16, header hygiene, changed-line clang-tidy and git diff --check passed. BE unit compilation was stopped at the user's request before execution; no BE unit pass result is claimed. Regression tests were not run at the user's request. Existing macOS ANN test portability adjustments used locally are excluded from this commit. - Behavior changed: Yes (explicit changelog protocol contract and transaction-bound recovery snapshots; ordinary writes do not require mappings). - Does this need documentation: No (local design and implementation notes are excluded from this code commit). --- be/src/agent/task_worker_pool.cpp | 9 +- .../cloud_engine_calc_delete_bitmap_task.cpp | 18 +- be/src/cloud/cloud_rowset_builder.cpp | 8 + be/src/cloud/cloud_schema_change_job.cpp | 3 +- be/src/cloud/cloud_tablet.cpp | 4 - be/src/exec/scan/olap_scanner.cpp | 75 +++++ be/src/exec/scan/olap_scanner.h | 8 + be/src/storage/changelog.h | 26 ++ be/src/storage/data_dir.cpp | 12 +- be/src/storage/iterator/block_reader.cpp | 13 +- .../iterator/vertical_block_reader.cpp | 3 +- be/src/storage/olap_server.cpp | 42 ++- be/src/storage/rowset_builder.cpp | 11 + be/src/storage/rowset_builder.h | 4 + be/src/storage/schema.cpp | 154 ++++------- be/src/storage/schema.h | 29 +- .../storage/schema_change/schema_change.cpp | 3 +- be/src/storage/storage_engine.h | 11 +- be/src/storage/tablet/base_tablet.cpp | 39 +-- be/src/storage/tablet/base_tablet.h | 10 +- be/src/storage/tablet/tablet.cpp | 5 - be/src/storage/tablet_info.cpp | 24 +- be/src/storage/tablet_info.h | 1 + .../task/engine_publish_version_task.cpp | 91 +++++-- .../task/engine_publish_version_task.h | 21 +- be/src/storage/transform/changelog_derive.cpp | 256 +++++++++++++----- be/src/storage/transform/changelog_derive.h | 10 +- be/src/storage/txn/txn_manager.cpp | 41 ++- be/src/storage/txn/txn_manager.h | 23 +- be/test/exec/scan/olap_scanner_test.cpp | 175 ++++++++++++ ...cloud_group_rowset_builder_writer_test.cpp | 38 +++ .../olap/rowset/group_rowset_builder_test.cpp | 34 +++ .../olap/rowset/group_rowset_writer_test.cpp | 8 +- .../changelog_vmerge_compaction_test.cpp | 3 +- .../ordered_data_compaction_test.cpp | 3 +- .../compaction/segcompaction_mow_test.cpp | 3 +- .../storage/compaction/segcompaction_test.cpp | 12 +- .../compaction/vertical_compaction_test.cpp | 27 +- .../block_reader_change_next_block_test.cpp | 20 +- ...k_reader_changelog_vcollect_merge_test.cpp | 15 +- be/test/storage/mow/mow_transform_test_base.h | 6 +- be/test/storage/read_schema_test.cpp | 114 ++++---- be/test/storage/rowid_conversion_test.cpp | 9 +- .../rowset/segment_flusher_format_test.cpp | 1 + be/test/storage/storage_engine_test.cpp | 75 ++++- .../transform/changelog_derive_test.cpp | 141 ++++++++-- .../storage/transform/validate_stage_test.cpp | 10 +- be/test/storage/txn/txn_manager_test.cpp | 63 ++++- .../variant_column_writer_reader_test.cpp | 3 +- .../variant_doc_mode_compaction_test.cpp | 6 +- be/test/testutil/creators.h | 65 ++++- be/test/testutil/index_storage_test_util.cpp | 3 +- .../catalog/LocalTabletInvertedIndex.java | 20 +- .../catalog/OlapChangelogTableWrapper.java | 23 ++ .../org/apache/doris/catalog/OlapTable.java | 4 +- .../doris/catalog/TabletInvertedIndex.java | 4 +- .../load/routineload/RoutineLoadJob.java | 2 +- .../kafka/KafkaRoutineLoadJob.java | 4 +- .../load/routineload/kafka/KafkaTaskInfo.java | 5 +- .../kinesis/KinesisRoutineLoadJob.java | 4 +- .../routineload/kinesis/KinesisTaskInfo.java | 5 +- .../apache/doris/master/ReportHandler.java | 18 +- .../translator/PhysicalPlanTranslator.java | 36 ++- .../load/NereidsLoadPlanInfoCollector.java | 6 + .../load/NereidsRoutineLoadTaskInfo.java | 10 +- .../commands/insert/OlapInsertExecutor.java | 5 + .../insert/RemoteOlapInsertExecutor.java | 9 +- .../trees/plans/logical/LogicalOlapScan.java | 12 +- .../apache/doris/planner/OlapScanNode.java | 29 ++ .../apache/doris/planner/OlapTableSink.java | 21 ++ .../doris/service/FrontendServiceImpl.java | 15 + .../apache/doris/task/PublishVersionTask.java | 42 +++ .../transaction/ChangelogWriteMapping.java | 175 ++++++++++++ .../transaction/DatabaseTransactionMgr.java | 13 +- .../GlobalTransactionMgrIface.java | 16 ++ .../transaction/PublishVersionDaemon.java | 1 + .../doris/transaction/TransactionState.java | 76 +++++- .../catalog/OlapTableChangelogSchemaTest.java | 16 ++ .../PhysicalPlanTranslatorTest.java | 54 ++++ .../load/VariantLoadParseInjectionTest.java | 2 +- .../ChangelogWriteMappingTest.java | 239 ++++++++++++++++ .../DatabaseTransactionMgrTest.java | 67 ++++- gensrc/proto/descriptors.proto | 1 + gensrc/proto/olap_file.proto | 12 + gensrc/thrift/AgentService.thrift | 2 + gensrc/thrift/Descriptors.thrift | 27 ++ gensrc/thrift/FrontendService.thrift | 2 + gensrc/thrift/PlanNodes.thrift | 1 + 88 files changed, 2234 insertions(+), 532 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/transaction/ChangelogWriteMapping.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/transaction/ChangelogWriteMappingTest.java diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index ab51e93e650da4..f8c5c04259a4a5 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -2186,8 +2186,13 @@ void PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest& } for (auto& item : discontinuous_version_tablets) { - _engine.add_async_publish_task(item.partition_id, item.tablet_id, item.publish_version, - publish_version_req.transaction_id, false, item.commit_tso); + auto enqueue_status = _engine.add_async_publish_task( + item.partition_id, item.tablet_id, item.publish_version, + publish_version_req.transaction_id, false, item.commit_tso, item.column_mappings); + if (!enqueue_status.ok()) { + status = std::move(enqueue_status); + error_tablet_ids.insert(item.tablet_id); + } } TFinishTaskRequest finish_task_request; if (!status.ok()) [[unlikely]] { diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index cd22c86800fc4c..9d5a1287e02aac 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -28,6 +28,7 @@ #include "cloud/cloud_tablet.h" #include "common/status.h" #include "runtime/memory/mem_tracker_limiter.h" +#include "storage/changelog.h" #include "storage/olap_common.h" #include "storage/rowset/beta_rowset.h" #include "storage/rowset/rowset.h" @@ -351,6 +352,18 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( return status; } + std::shared_ptr changelog_layout; + if (attach_changelog.rowset != nullptr) { + if (attach_changelog.column_mappings == nullptr) { + return Status::InvalidArgument("cloud txn {} tablet {} misses changelog mappings", + transaction_id, _tablet_id); + } + RETURN_IF_ERROR(changelog::resolve_layout( + rowset->tablet_schema()->copy_without_variant_extracted_columns(), + attach_changelog.rowset->tablet_schema()->copy_without_variant_extracted_columns(), + *attach_changelog.column_mappings, &changelog_layout)); + } + rowset->set_version(Version(version, version)); if (rowset->tablet_schema()->is_tso_enabled() && rowset->commit_tso().start_tso() == -1) { DORIS_CHECK_EQ(rowset->commit_tso().end_tso(), -1); @@ -406,14 +419,15 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( if (invisible_rowsets == nullptr) { status = CloudTablet::update_delete_bitmap(tablet, &txn_info, transaction_id, - txn_expiration); + txn_expiration, nullptr, changelog_layout); } else { txn_info.is_txn_load = true; txn_info.invisible_rowsets = *invisible_rowsets; txn_info.lock_id = _transaction_id; txn_info.next_visible_version = _version; status = CloudTablet::update_delete_bitmap(tablet, &txn_info, transaction_id, - txn_expiration, tablet_delete_bitmap); + txn_expiration, tablet_delete_bitmap, + changelog_layout); } } if (status != Status::OK()) { diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index d8b5971252edaa..de4c4c0b5d3300 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -150,6 +150,13 @@ Status CloudGroupRowsetBuilder::init() { auto& changelog_ctx = const_cast(_changelog_builder->rowset_writer()->context()); auto& cfg = changelog_ctx.write_changelog_opt().write_changelog_config(); + const auto& mapping = _data_builder->changelog_column_mappings(); + if (mapping == nullptr) { + return Status::InvalidArgument("source index misses changelog column mappings"); + } + RETURN_IF_ERROR(changelog::resolve_layout( + data_ctx.tablet_schema, changelog_ctx.tablet_schema, *mapping, &cfg.layout)); + cfg.write_before = cfg.layout->need_historical_value; cfg.source.tablet_schema = data_ctx.tablet_schema; cfg.source.partial_update_info = data_ctx.partial_update_info; cfg.source.mow_context = data_ctx.mow_context; @@ -194,6 +201,7 @@ Status CloudGroupRowsetBuilder::commit_rowset(const std::string& job_id, int64_t Status CloudGroupRowsetBuilder::set_txn_related_info() { ChangelogTxnInfo attach_changelog; + attach_changelog.column_mappings = _data_builder->changelog_column_mappings(); attach_changelog.rowset = _changelog_builder->rowset(); attach_changelog.tablet = _changelog_builder->tablet_sptr(); if (_data_builder->tablet()->enable_unique_key_merge_on_write()) { diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index a926d60b40c1e0..13fd2e13290965 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -245,8 +245,7 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque ReadSchemaSPtr read_schema = std::make_shared(_base_tablet_schema->columns()); RETURN_IF_ERROR(read_schema->init_from_tablet_schema(*_base_tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false)); + /*merge_by_sequence_mapping=*/false)); // delete handlers to filter out deleted rows DeleteHandler delete_handler; diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 43b887d0f51a62..e943eac746eed8 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -850,9 +850,6 @@ Result> CloudTablet::create_rowset_writer( context.partition_id = partition_id(); context.enable_unique_key_merge_on_write = enable_unique_key_merge_on_write(); context.encrypt_algorithm = tablet_meta()->encryption_algorithm(); - if (context.write_changelog_opt().enable) { - context.write_changelog_opt().set_need_before(binlog_config().need_historical_value()); - } context.inverted_index_storage_format = tablet_meta()->inverted_index_storage_format(); context.persist_inverted_index_storage_format = tablet_meta()->has_inverted_index_storage_format(); @@ -895,7 +892,6 @@ Result> CloudTablet::create_transient_rowset_write context.is_transient_rowset_writer = true; if (rowset.rowset_meta() != nullptr && rowset.rowset_meta()->is_changelog()) { context.write_changelog_opt().enable = true; - context.write_changelog_opt().set_need_before(binlog_config().need_historical_value()); } context.rowset_id = rowset.rowset_id(); context.tablet_id = tablet_id(); diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index eb0d1604f609dd..283c774821261a 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet_hotspot.h" @@ -656,7 +657,81 @@ Status OlapScanner::_init_read_schema() { // every storage key, sequence, TSO and changelog column required below. _tablet_reader_params.read_schema = std::make_shared(std::move(read_columns), std::move(expected_types)); + const auto& scan_node = assert_cast(_local_state)->olap_scan_node(); + return _init_changelog_read_mappings( + scan_node, _output_tuple_desc->slots(), *_tablet_reader_params.tablet_schema, + _tablet_reader_params.changelog_scan_type, *_tablet_reader_params.read_schema); +} +Status OlapScanner::_init_changelog_read_mappings(const TOlapScanNode& scan_node, + const std::vector& slots, + const TabletSchema& tablet_schema, + TChangelogScanType::type scan_type, + ReadSchema& read_schema) { + const bool merge_scan = + scan_type == TChangelogScanType::DETAIL || scan_type == TChangelogScanType::MIN_DELTA; + // Missing mappings are a malformed merge plan. Explicit empty pairs are valid for + // key-only reads and DETAIL on a table that does not record historical values. + if (merge_scan && (!scan_node.__isset.changelog_column_mappings || + !scan_node.changelog_column_mappings.__isset.pairs)) { + return Status::InvalidArgument("Changelog merge scan requires explicit read mappings"); + } + DORIS_CHECK_EQ(slots.size(), read_schema.num_block_columns()); + std::unordered_map slot_ordinals; + for (ColumnId ordinal = 0; ordinal < slots.size(); ++ordinal) { + if (!slot_ordinals.emplace(slots[ordinal]->id(), ordinal).second) { + return Status::InvalidArgument("Duplicate changelog scan slot {}", + slots[ordinal]->id()); + } + } + ReadSchema::ChangelogValueColumnPairs pairs; + if (scan_node.__isset.changelog_column_mappings) { + for (const auto& pair : scan_node.changelog_column_mappings.pairs) { + if (!pair.__isset.current_slot_id || !pair.__isset.before_slot_id) { + return Status::InvalidArgument("Changelog read pair requires both slot IDs"); + } + const auto current = slot_ordinals.find(pair.current_slot_id); + const auto before = slot_ordinals.find(pair.before_slot_id); + if (current == slot_ordinals.end() || before == slot_ordinals.end()) { + return Status::InvalidArgument("Unknown changelog read slot pair ({}, {})", + pair.current_slot_id, pair.before_slot_id); + } + pairs.emplace_back(current->second, before->second); + } + } + RETURN_IF_ERROR(read_schema.init_changelog_column_mappings(tablet_schema, pairs)); + if (scan_type != TChangelogScanType::NONE && + (read_schema.tso_ordinal() < 0 || read_schema.op_ordinal() < 0)) { + return Status::InvalidArgument("Changelog scan requires TSO and OP columns"); + } + if (scan_type == TChangelogScanType::MIN_DELTA) { + std::unordered_set paired; + for (const auto& [current, before] : pairs) { + paired.insert(current); + paired.insert(before); + } + for (ColumnId cid = 0; cid < tablet_schema.num_columns(); ++cid) { + if (static_cast(cid) == tablet_schema.changelog_tso_col_idx() || + static_cast(cid) == tablet_schema.changelog_lsn_col_idx() || + static_cast(cid) == tablet_schema.changelog_op_col_idx()) { + continue; + } + const auto& column = tablet_schema.column(cid); + const int32_t ordinal = read_schema.ordinal_by_uid(column.unique_id()); + // Even an unprojected value may change: comparing only v1 would incorrectly + // suppress an UPDATE that changed v2. Require the entire original value shape. + if (ordinal < 0 || static_cast(ordinal) >= read_schema.num_block_columns() || + (!column.is_key() && !paired.contains(cast_set(ordinal))) || + !remove_nullable(read_schema.data_type(ordinal)) + ->equals(*remove_nullable(column.get_vec_type())) || + remove_nullable(read_schema.data_type(ordinal))->get_name() != + remove_nullable(column.get_vec_type())->get_name()) { + return Status::InvalidArgument( + "MIN_DELTA requires complete current and before values: column uid {}", + column.unique_id()); + } + } + } return Status::OK(); } diff --git a/be/src/exec/scan/olap_scanner.h b/be/src/exec/scan/olap_scanner.h index 4a4f45e0a88096..587a5f8e565acf 100644 --- a/be/src/exec/scan/olap_scanner.h +++ b/be/src/exec/scan/olap_scanner.h @@ -47,6 +47,9 @@ struct OlapScanRange; class RuntimeProfile; class RuntimeState; class TPaloScanRange; +class TOlapScanNode; +class ReadSchema; +class SlotDescriptor; class ScanLocalStateBase; struct FilterPredicates; #ifndef NDEBUG @@ -118,6 +121,11 @@ class OlapScanner : public Scanner { [[nodiscard]] Status _init_tso_predicates(); [[nodiscard]] Status _init_read_schema(); + static Status _init_changelog_read_mappings(const TOlapScanNode& scan_node, + const std::vector& slots, + const TabletSchema& tablet_schema, + TChangelogScanType::type scan_type, + ReadSchema& read_schema); [[nodiscard]] Status _init_variant_columns(); #ifndef NDEBUG Status _check_ann_cache_hit_debug_points(const OlapReaderStatistics& stats); diff --git a/be/src/storage/changelog.h b/be/src/storage/changelog.h index a8c2c045fb0c1d..23c943902bc4cc 100644 --- a/be/src/storage/changelog.h +++ b/be/src/storage/changelog.h @@ -40,11 +40,36 @@ namespace doris { class AutoIncIDBuffer; +class PChangelogWriteColumnMappings; +class TChangelogWriteColumnMappings; struct PartialUpdateInfo; struct MowContext; namespace changelog { +// A layout belongs to exactly these two pinned schemas. Writers share it for +// their operation; no CID is stored in rowset metadata or reused after recovery. +struct ResolvedChangelogLayout { + TabletSchemaSPtr source_schema; + TabletSchemaSPtr target_schema; + std::vector current_source_cids; + std::vector current_target_cids; + std::vector current_nullable_widening; + std::vector before_source_cids; + std::vector before_target_cids; + uint32_t tso_cid; + uint32_t lsn_cid; + uint32_t op_cid; + bool need_historical_value; +}; + +Status validate_mapping(const PChangelogWriteColumnMappings& mapping); +Status mapping_from_thrift(const TChangelogWriteColumnMappings& input, + PChangelogWriteColumnMappings* output); +Status resolve_layout(const TabletSchemaSPtr& source, const TabletSchemaSPtr& target, + const PChangelogWriteColumnMappings& mapping, + std::shared_ptr* result); + // ATTN(changelog-rename): formerly "__BEFORE____" inline std::string build_before_column_name(std::string_view name) { std::string before_name = "__DORIS_CHANGELOG_BEFORE_"; @@ -144,6 +169,7 @@ class SegmentAllocatedLsnMap { struct SegmentWriteChangelogOptions { public: bool write_before = false; + std::shared_ptr layout; // source context, used for retrieving historical row and building changelog block struct SourceWriteDataOptions { diff --git a/be/src/storage/data_dir.cpp b/be/src/storage/data_dir.cpp index 4f4d12ca9296c0..a07290cb72c6ae 100644 --- a/be/src/storage/data_dir.cpp +++ b/be/src/storage/data_dir.cpp @@ -495,9 +495,15 @@ Status DataDir::load() { LOG(WARNING) << "parse pending publish info failed, tablet_id: " << tablet_id << " publish_version: " << publish_version; } - engine.add_async_publish_task(pending_publish_info_pb.partition_id(), tablet_id, - publish_version, pending_publish_info_pb.transaction_id(), - true, pending_publish_info_pb.commit_tso()); + CHECK(engine.add_async_publish_task( + pending_publish_info_pb.partition_id(), tablet_id, publish_version, + pending_publish_info_pb.transaction_id(), true, + pending_publish_info_pb.commit_tso(), + pending_publish_info_pb.has_changelog_column_mappings() + ? std::make_shared( + pending_publish_info_pb.changelog_column_mappings()) + : nullptr) + .ok()); return true; }; MonotonicStopWatch pending_publish_timer; diff --git a/be/src/storage/iterator/block_reader.cpp b/be/src/storage/iterator/block_reader.cpp index 44c1b10f52bbba..338b908e23ecbf 100644 --- a/be/src/storage/iterator/block_reader.cpp +++ b/be/src/storage/iterator/block_reader.cpp @@ -555,19 +555,18 @@ Status BlockReader::init(const ReaderParams& read_params) { SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns); RETURN_IF_ERROR(TabletReader::init(read_params)); - // A changelog scan maps the before-image columns; every other read of this reader, which is - // the one that merges rows across rowsets, builds the sequence mapping instead. - const bool map_changelog_columns = + // Scanner already resolved changelog pairs. Other merge reads may need a sequence mapping. + const bool is_changelog_merge_scan = read_params.changelog_scan_type == TChangelogScanType::MIN_DELTA || read_params.changelog_scan_type == TChangelogScanType::DETAIL; const bool merge_by_sequence_mapping = - !map_changelog_columns && _tablet_schema->has_seq_map() && + !is_changelog_merge_scan && _tablet_schema->has_seq_map() && _tablet_schema->keys_type() == UNIQUE_KEYS && !_direct_mode && !(read_params.reader_type == ReaderType::READER_QUERY && _tablet->enable_unique_key_merge_on_write()); - RETURN_IF_ERROR(_read_schema->init_from_tablet_schema( - *_tablet_schema, merge_by_sequence_mapping, map_changelog_columns)); - if (map_changelog_columns) { + RETURN_IF_ERROR( + _read_schema->init_from_tablet_schema(*_tablet_schema, merge_by_sequence_mapping)); + if (is_changelog_merge_scan) { _min_delta_value_compare_unsupported = false; } diff --git a/be/src/storage/iterator/vertical_block_reader.cpp b/be/src/storage/iterator/vertical_block_reader.cpp index 2913b144cf94c3..0f6c2cf86015ba 100644 --- a/be/src/storage/iterator/vertical_block_reader.cpp +++ b/be/src/storage/iterator/vertical_block_reader.cpp @@ -270,8 +270,7 @@ Status VerticalBlockReader::init(const ReaderParams& read_params, RETURN_IF_ERROR(TabletReader::init(read_params)); RETURN_IF_ERROR(_read_schema->init_from_tablet_schema(*_tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false)); + /*merge_by_sequence_mapping=*/false)); auto status = _init_collect_iter(read_params, sample_info); if (!status.ok()) [[unlikely]] { diff --git a/be/src/storage/olap_server.cpp b/be/src/storage/olap_server.cpp index a1e74d2968e64d..069d884db9d63e 100644 --- a/be/src/storage/olap_server.cpp +++ b/be/src/storage/olap_server.cpp @@ -1706,9 +1706,10 @@ void StorageEngine::_follow_cooldown_meta(TabletSharedPtr t) { } } -void StorageEngine::add_async_publish_task(int64_t partition_id, int64_t tablet_id, - int64_t publish_version, int64_t transaction_id, - bool is_recovery, int64_t commit_tso) { +Status StorageEngine::add_async_publish_task( + int64_t partition_id, int64_t tablet_id, int64_t publish_version, int64_t transaction_id, + bool is_recovery, int64_t commit_tso, + std::shared_ptr column_mappings) { if (!is_recovery) { bool exists = false; { @@ -1722,27 +1723,34 @@ void StorageEngine::add_async_publish_task(int64_t partition_id, int64_t tablet_ } } if (exists) { - return; + return Status::OK(); } TabletSharedPtr tablet = tablet_manager()->get_tablet(tablet_id); if (tablet == nullptr) { LOG(INFO) << "tablet may be dropped when add async publish task, tablet_id: " << tablet_id; - return; + return Status::OK(); } PendingPublishInfoPB pending_publish_info_pb; pending_publish_info_pb.set_partition_id(partition_id); pending_publish_info_pb.set_transaction_id(transaction_id); pending_publish_info_pb.set_commit_tso(commit_tso); - static_cast(TabletMetaManager::save_pending_publish_info( - tablet->data_dir(), tablet->tablet_id(), publish_version, - pending_publish_info_pb.SerializeAsString())); + if (column_mappings != nullptr) { + *pending_publish_info_pb.mutable_changelog_column_mappings() = *column_mappings; + } + RETURN_IF_ERROR(SYNC_POINT_HOOK_RETURN_VALUE( + TabletMetaManager::save_pending_publish_info( + tablet->data_dir(), tablet->tablet_id(), publish_version, + pending_publish_info_pb.SerializeAsString()), + "StorageEngine::add_async_publish_task::save_pending_publish_info")); } LOG(INFO) << "add pending publish task, tablet_id: " << tablet_id << " version: " << publish_version << " txn_id:" << transaction_id << " is_recovery: " << is_recovery; std::unique_lock wlock(_async_publish_lock); - _async_publish_tasks[tablet_id][publish_version] = {transaction_id, partition_id, commit_tso}; + _async_publish_tasks[tablet_id][publish_version] = {transaction_id, partition_id, commit_tso, + std::move(column_mappings)}; + return Status::OK(); } int64_t StorageEngine::get_pending_publish_min_version(int64_t tablet_id) { @@ -1803,11 +1811,19 @@ void StorageEngine::_process_async_publish() { } auto async_publish_task = std::make_shared( - *this, tablet, partition_id, transaction_id, version, commit_tso); - static_cast(_tablet_publish_txn_thread_pool->submit_func( - [=]() { async_publish_task->handle(); })); + *this, tablet, partition_id, transaction_id, version, commit_tso, + std::get<3>(task_iter->second)); + auto submit_status = _tablet_publish_txn_thread_pool->submit_func( + [=]() { async_publish_task->handle(); }); + // A stopped/full executor has not taken ownership. Keep the queue + // entry and its durable snapshot for the next scheduling attempt. + if (!submit_status.ok()) { + LOG(WARNING) << "submit async publish failed, tablet_id=" << tablet_id + << ", version=" << version << ", status=" << submit_status; + ++tablet_iter; + continue; + } tablet_iter->second.erase(task_iter); - need_removed_tasks.emplace_back(tablet, version); tablet_iter++; } } diff --git a/be/src/storage/rowset_builder.cpp b/be/src/storage/rowset_builder.cpp index eeede858596115..d3b041424082d8 100644 --- a/be/src/storage/rowset_builder.cpp +++ b/be/src/storage/rowset_builder.cpp @@ -447,6 +447,9 @@ Status BaseRowsetBuilder::_build_current_tablet_schema( } } + DCHECK(index_schema != nullptr); + _changelog_column_mappings = index_schema->changelog_column_mappings; + auto cache_key = TabletSchemaCache::build_load_schema_cache_key( index_id, table_schema_param, ori_tablet_schema, index_schema); auto cached_schema = TabletSchemaCache::instance()->lookup_schema(cache_key); @@ -582,6 +585,13 @@ Status GroupRowsetBuilder::init() { auto& changelog_ctx = const_cast( _changelog_rowset_builder->rowset_writer()->context()); auto& cfg = changelog_ctx.write_changelog_opt().write_changelog_config(); + const auto& mapping = _txn_rs_builder->changelog_column_mappings(); + if (mapping == nullptr) { + return Status::InvalidArgument("source index misses changelog column mappings"); + } + RETURN_IF_ERROR(changelog::resolve_layout( + data_ctx.tablet_schema, changelog_ctx.tablet_schema, *mapping, &cfg.layout)); + cfg.write_before = cfg.layout->need_historical_value; cfg.source.tablet_schema = data_ctx.tablet_schema; cfg.source.partial_update_info = data_ctx.partial_update_info; cfg.source.mow_context = data_ctx.mow_context; @@ -613,6 +623,7 @@ Status GroupRowsetBuilder::commit_txn() { // Attach changelog rowset (and its independent changelog tablet) to txn rowset, so that // commit/rollback and clean-up are all handled by txn rowset builder. ChangelogTxnInfo attach_changelog; + attach_changelog.column_mappings = _txn_rs_builder->changelog_column_mappings(); attach_changelog.rowset = _changelog_rowset_builder->rowset(); attach_changelog.tablet = std::static_pointer_cast(_changelog_rowset_builder->tablet_sptr()); diff --git a/be/src/storage/rowset_builder.h b/be/src/storage/rowset_builder.h index de83888a679d39..dffa98d8398c4f 100644 --- a/be/src/storage/rowset_builder.h +++ b/be/src/storage/rowset_builder.h @@ -89,6 +89,9 @@ class BaseRowsetBuilder { bool is_data_builder() const { return _req.write_req_type == WriteRequestType::DATA; } // Attach the changelog rowset and its independent changelog tablet to the same txn. + const std::shared_ptr& changelog_column_mappings() const { + return _changelog_column_mappings; + } Status attach_changelog_to_txn(const ChangelogTxnInfo& attach_changelog); // Attach an extra pending rowset id so that PendingLocalRowsets can be @@ -116,6 +119,7 @@ class BaseRowsetBuilder { RowsetSharedPtr _rowset; // The changelog rowset and its independent changelog tablet attached to the same txn. ChangelogTxnInfo _attach_changelog; + std::shared_ptr _changelog_column_mappings; std::shared_ptr _rowset_writer; PendingRowsetGuard _pending_rs_guard; // Extra rowset ids that share the same PendingRowsetGuard. diff --git a/be/src/storage/schema.cpp b/be/src/storage/schema.cpp index 1d6df92e2c1435..e352fb8ed574e1 100644 --- a/be/src/storage/schema.cpp +++ b/be/src/storage/schema.cpp @@ -37,26 +37,11 @@ #include "core/data_type/data_type_variant_v2.h" #include "core/data_type/primitive_type.h" #include "runtime/descriptors.h" -#include "storage/changelog.h" namespace doris { namespace { -bool changelog_value_columns_have_same_type(const TabletColumn& lhs, const TabletColumn& rhs) { - if (lhs.type() != rhs.type() || lhs.is_nullable() != rhs.is_nullable() || - lhs.length() != rhs.length() || lhs.precision() != rhs.precision() || - lhs.frac() != rhs.frac() || lhs.get_subtype_count() != rhs.get_subtype_count()) { - return false; - } - for (uint32_t i = 0; i < lhs.get_subtype_count(); ++i) { - if (!changelog_value_columns_have_same_type(lhs.get_sub_column(i), rhs.get_sub_column(i))) { - return false; - } - } - return true; -} - // Sets the type of `column` from `data_type`, sub columns included. Sub columns get the names FE // gives them, and the length is computed the same way as for a column FE sends. void set_type_from_data_type(const DataTypePtr& data_type, TabletColumn* column) { @@ -190,96 +175,76 @@ void ReadSchema::append_compaction_rowid_column() { column->set_name(BeConsts::COMPACTION_ROWID_COL); _read_columns.emplace_back(column); _read_types.emplace_back(column->get_vec_type()); + _rowid_ordinal = cast_set(_num_block_columns); + _uid_to_ordinal.emplace(column->unique_id(), _rowid_ordinal); + _before_column_ordinals.push_back(cast_set(_num_block_columns)); ++_num_block_columns; - _init_descriptors(); } -void ReadSchema::_init_before_column_ordinals() { - std::unordered_map name_to_ordinal; - name_to_ordinal.reserve(_num_block_columns); - for (ColumnId ordinal = 0; ordinal < _num_block_columns; ++ordinal) { - name_to_ordinal.emplace(_read_columns[ordinal]->name(), ordinal); - } - - _before_column_ordinals.resize(_num_block_columns); - for (ColumnId ordinal = 0; ordinal < _num_block_columns; ++ordinal) { - auto read_ordinal = static_cast(ordinal); - if (read_ordinal == _tso_ordinal || read_ordinal == _lsn_ordinal || - read_ordinal == _op_ordinal) { - _before_column_ordinals[ordinal] = ordinal; - continue; +Status ReadSchema::init_changelog_column_mappings(const ChangelogValueColumnPairs& pairs, + int32_t tso_ordinal, int32_t lsn_ordinal, + int32_t op_ordinal) { + std::vector used(_num_block_columns, false); + for (int32_t ordinal : {tso_ordinal, lsn_ordinal, op_ordinal}) { + // -1 is the only absent-column sentinel; private delete columns cannot be in a Block. + if (ordinal < -1 || (ordinal >= 0 && static_cast(ordinal) >= _num_block_columns)) { + return Status::InvalidArgument("Invalid changelog system ordinal {}", ordinal); } - auto before_name = changelog::build_before_column_name(_read_columns[ordinal]->name()); - auto before = name_to_ordinal.find(before_name); - _before_column_ordinals[ordinal] = - before == name_to_ordinal.end() ? ordinal : before->second; - } -} - -void ReadSchema::_init_changelog_column_mappings(const TabletSchema& tablet_schema) { - DORIS_CHECK_GE(_op_ordinal, 0); - DORIS_CHECK_EQ(_before_column_ordinals.size(), _num_block_columns); - - _changelog_value_column_pairs.clear(); - _changelog_value_pairs_complete = false; - - std::vector value_column_ids; - value_column_ids.reserve(tablet_schema.num_columns() - tablet_schema.num_key_columns()); - for (ColumnId cid = 0; cid < tablet_schema.num_columns(); ++cid) { - if (static_cast(cid) == tablet_schema.changelog_tso_col_idx() || - static_cast(cid) == tablet_schema.changelog_lsn_col_idx() || - static_cast(cid) == tablet_schema.changelog_op_col_idx() || - tablet_schema.column(cid).is_key()) { + if (ordinal == -1) { continue; } - value_column_ids.push_back(cid); - } - - if (value_column_ids.empty() || value_column_ids.size() % 2 != 0) { - return; + if (used[ordinal] || _read_columns[ordinal]->is_key() || + remove_nullable(_read_types[ordinal])->get_primitive_type() != TYPE_BIGINT) { + return Status::InvalidArgument("Invalid changelog system column at ordinal {}", + ordinal); + } + used[ordinal] = true; } - - const size_t value_column_count = value_column_ids.size() / 2; - for (size_t i = 0; i < value_column_count; ++i) { - const auto& after = tablet_schema.column(value_column_ids[i]); - const auto& before = tablet_schema.column(value_column_ids[i + value_column_count]); - if (before.name() != changelog::build_before_column_name(after.name()) || - !changelog_value_columns_have_same_type(after, before)) { - return; + for (const auto& [current, before] : pairs) { + // Reusing either endpoint, including a system/key column, corrupts BEFORE materialization. + if (current >= _num_block_columns || before >= _num_block_columns || current == before || + used[current] || used[before] || _read_columns[current]->is_key() || + _read_columns[before]->is_key() || + !_read_types[current]->equals(*_read_types[before]) || + _read_types[current]->get_name() != _read_types[before]->get_name()) { + return Status::InvalidArgument("Invalid changelog value pair ({}, {})", current, + before); } + used[current] = true; + used[before] = true; } - - // A valid physical changelog layout starts from an identity mapping. Only AFTER value - // columns map to their BEFORE companions; keys, metadata and BEFORE columns map to themselves. + _tso_ordinal = tso_ordinal; + _lsn_ordinal = lsn_ordinal; + _op_ordinal = op_ordinal; + _before_column_ordinals.resize(_num_block_columns); for (ColumnId ordinal = 0; ordinal < _num_block_columns; ++ordinal) { _before_column_ordinals[ordinal] = ordinal; } - - bool complete = true; - _changelog_value_column_pairs.reserve(value_column_count); - for (size_t i = 0; i < value_column_count; ++i) { - const auto after_cid = value_column_ids[i]; - const auto before_cid = value_column_ids[i + value_column_count]; - const int32_t after_ordinal = ordinal_by_uid(tablet_schema.column(after_cid).unique_id()); - const int32_t before_ordinal = ordinal_by_uid(tablet_schema.column(before_cid).unique_id()); - if (after_ordinal < 0 || before_ordinal < 0 || - static_cast(after_ordinal) >= _num_block_columns || - static_cast(before_ordinal) >= _num_block_columns) { - complete = false; - continue; + _changelog_value_column_pairs = pairs; + for (const auto& [current, before] : pairs) { + _before_column_ordinals[current] = before; + } + _changelog_value_pairs_complete = true; + for (ColumnId ordinal = 0; ordinal < _num_block_columns; ++ordinal) { + if (!_read_columns[ordinal]->is_key() && !used[ordinal]) { + _changelog_value_pairs_complete = false; } + } + return Status::OK(); +} - const auto after = cast_set(after_ordinal); - const auto before = cast_set(before_ordinal); - _before_column_ordinals[after] = before; - if (!_read_types[after]->equals(*_read_types[before])) { - complete = false; - continue; +Status ReadSchema::init_changelog_column_mappings(const TabletSchema& tablet_schema, + const ChangelogValueColumnPairs& pairs) { + const auto resolve = [&](int32_t cid) -> int32_t { + if (cid < 0) { + return -1; } - _changelog_value_column_pairs.emplace_back(after, before); - } - _changelog_value_pairs_complete = - complete && _changelog_value_column_pairs.size() == value_column_count; + const int32_t ordinal = ordinal_by_uid(tablet_schema.column(cid).unique_id()); + return ordinal >= 0 && static_cast(ordinal) < _num_block_columns ? ordinal : -1; + }; + return init_changelog_column_mappings(pairs, resolve(tablet_schema.changelog_tso_col_idx()), + resolve(tablet_schema.changelog_lsn_col_idx()), + resolve(tablet_schema.changelog_op_col_idx())); } Block ReadSchema::create_read_block() const { @@ -320,9 +285,7 @@ std::string ReadSchema::read_columns_to_string() const { } Status ReadSchema::init_from_tablet_schema(const TabletSchema& tablet_schema, - bool merge_by_sequence_mapping, - bool map_changelog_columns) { - DORIS_CHECK(!(merge_by_sequence_mapping && map_changelog_columns)); + bool merge_by_sequence_mapping) { _tablet_has_sequence_map = tablet_schema.has_seq_map(); _tablet_has_extracted_variant_columns = std::ranges::any_of(tablet_schema.columns(), @@ -330,10 +293,7 @@ Status ReadSchema::init_from_tablet_schema(const TabletSchema& tablet_schema, if (merge_by_sequence_mapping) { RETURN_IF_ERROR(_init_sequence_map(tablet_schema)); } - if (map_changelog_columns) { - _init_changelog_column_mappings(tablet_schema); - } - return Status::OK(); + return init_changelog_column_mappings(tablet_schema, _changelog_value_column_pairs); } Status ReadSchema::_init_sequence_map(const TabletSchema& tablet_schema) { diff --git a/be/src/storage/schema.h b/be/src/storage/schema.h index 540bbc1aa1d12c..828ff7fb53ab14 100644 --- a/be/src/storage/schema.h +++ b/be/src/storage/schema.h @@ -94,10 +94,17 @@ class ReadSchema { std::string read_columns_to_string() const; - // Always sets the two facts below; the flags add the sequence mapping and the changelog - // before-image mapping. + // Sets tablet facts and resolves special-column UIDs, without deriving value relationships. Status init_from_tablet_schema(const TabletSchema& tablet_schema, - bool merge_by_sequence_mapping, bool map_changelog_columns); + bool merge_by_sequence_mapping); + + // All positions address the fixed caller-visible Block prefix. -1 means a system column + // is absent. Pairs are explicit relationships, independent of column names and ordering. + Status init_changelog_column_mappings(const ChangelogValueColumnPairs& pairs, + int32_t tso_ordinal, int32_t lsn_ordinal, + int32_t op_ordinal); + Status init_changelog_column_mappings(const TabletSchema& tablet_schema, + const ChangelogValueColumnPairs& pairs); // Whether the tablet schema defines a sequence mapping -- not whether this read merges by // one. A MoW query does not merge, yet a value predicate still must not be pushed down. @@ -168,9 +175,7 @@ class ReadSchema { private: void _init_read_types(); - void _init_before_column_ordinals(); Status _init_sequence_map(const TabletSchema& tablet_schema); - void _init_changelog_column_mappings(const TabletSchema& tablet_schema); void _init_descriptors() { DORIS_CHECK_LE(_num_block_columns, _read_columns.size()); @@ -210,21 +215,13 @@ class ReadSchema { if (col.name() == VERSION_COL) { _version_ordinal = i; } - if (col.name() == CHANGELOG_TSO_COL) { - _tso_ordinal = i; - } - if (col.name() == CHANGELOG_LSN_COL) { - _lsn_ordinal = i; - } - if (col.name() == CHANGELOG_OP_COL) { - _op_ordinal = i; - } if (col.name() == COMMIT_TSO_COL) { _commit_tso_ordinal = i; } } - if (_op_ordinal >= 0) { - _init_before_column_ordinals(); + _before_column_ordinals.resize(_num_block_columns); + for (ColumnId ordinal = 0; ordinal < _num_block_columns; ++ordinal) { + _before_column_ordinals[ordinal] = ordinal; } } diff --git a/be/src/storage/schema_change/schema_change.cpp b/be/src/storage/schema_change/schema_change.cpp index 90f83baf76d098..c5bf873b242736 100644 --- a/be/src/storage/schema_change/schema_change.cpp +++ b/be/src/storage/schema_change/schema_change.cpp @@ -988,8 +988,7 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques _base_tablet_schema->columns().begin() + num_cols); ReadSchemaSPtr read_schema = std::make_shared(std::move(read_columns)); RETURN_IF_ERROR(read_schema->init_from_tablet_schema(*_base_tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false)); + /*merge_by_sequence_mapping=*/false)); std::vector cluster_key_idxes; DBUG_EXECUTE_IF("SchemaChangeJob::_do_process_alter_tablet.block", DBUG_BLOCK); diff --git a/be/src/storage/storage_engine.h b/be/src/storage/storage_engine.h index 221c3846653757..f7ac3836be951b 100644 --- a/be/src/storage/storage_engine.h +++ b/be/src/storage/storage_engine.h @@ -55,6 +55,7 @@ #include "util/countdown_latch.h" namespace doris { +class PChangelogWriteColumnMappings; class DataDir; class EngineTask; @@ -371,8 +372,10 @@ class StorageEngine final : public BaseStorageEngine { void gc_binlogs(const std::unordered_map& gc_tablet_infos); - void add_async_publish_task(int64_t partition_id, int64_t tablet_id, int64_t publish_version, - int64_t transaction_id, bool is_recover, int64_t commit_tso); + Status add_async_publish_task( + int64_t partition_id, int64_t tablet_id, int64_t publish_version, + int64_t transaction_id, bool is_recover, int64_t commit_tso, + std::shared_ptr column_mappings = nullptr); int64_t get_pending_publish_min_version(int64_t tablet_id); bool add_broken_path(std::string path); @@ -605,7 +608,9 @@ class StorageEngine final : public BaseStorageEngine { std::mutex _cumu_compaction_delay_mtx; // tablet_id, publish_version, transaction_id, partition_id, commit_tso - std::map>> + std::map>>> _async_publish_tasks; // aync publish for discontinuous versions of merge_on_write table std::shared_ptr _async_publish_thread; diff --git a/be/src/storage/tablet/base_tablet.cpp b/be/src/storage/tablet/base_tablet.cpp index eba76fe573b28f..14c933cabe6d48 100644 --- a/be/src/storage/tablet/base_tablet.cpp +++ b/be/src/storage/tablet/base_tablet.cpp @@ -1676,9 +1676,10 @@ Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap return Status::OK(); } -Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info, - int64_t txn_id, int64_t txn_expiration, - DeleteBitmapPtr tablet_delete_bitmap) { +Status BaseTablet::update_delete_bitmap( + const BaseTabletSPtr& self, TabletTxnInfo* txn_info, int64_t txn_id, int64_t txn_expiration, + DeleteBitmapPtr tablet_delete_bitmap, + std::shared_ptr changelog_layout) { SCOPED_BVAR_LATENCY(g_tablet_update_delete_bitmap_latency); RowsetIdUnorderedSet cur_rowset_ids; RowsetIdUnorderedSet rowset_ids_to_add; @@ -1692,31 +1693,15 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf bool is_partial_update = txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update(); const auto& changelog_rs = txn_info->attach_changelog.rowset; - if (changelog_rs != nullptr && changelog_rs->rowset_meta() != nullptr && - changelog_rs->rowset_meta()->is_changelog()) { + if (changelog_rs != nullptr) { DCHECK(txn_info->attach_changelog.tablet != nullptr); - changelog_rowset = changelog_rs; - build_changelog = - is_partial_update || - txn_info->attach_changelog.tablet->binlog_config().need_historical_value(); - } - // The rewrite below reads the LSN of conflicting rows from the attached changelog rowset and - // derives TSO/LSN/OP for new changelog segments, at positions found by the hidden column - // names. Example: a load committed by a build that named those columns differently, then - // published after the upgrade (the FE resends the publish after every restart): its rowset - // has none of them, and the LSN read would index past the schema. Fail the publish instead. - if (build_changelog) { - const auto& changelog_schema = *changelog_rowset->tablet_schema(); - if (changelog_schema.changelog_tso_col_idx() < 0 || - changelog_schema.changelog_lsn_col_idx() < 0 || - changelog_schema.changelog_op_col_idx() < 0) { - return Status::InternalError( - "changelog rowset {} misses its hidden columns {}, {} or {}: changelog " - "tablet={}, base tablet={}, txn_id={}", - changelog_rowset->rowset_id().to_string(), CHANGELOG_TSO_COL, CHANGELOG_LSN_COL, - CHANGELOG_OP_COL, txn_info->attach_changelog.tablet->tablet_id(), - self->tablet_id(), txn_id); + // Publishing validates before visibility and supplies the same schema-bound + // layout to this writer, including key-only historical transactions. + if (changelog_layout == nullptr) { + return Status::InvalidArgument("changelog bitmap update requires resolved layout"); } + changelog_rowset = changelog_rs; + build_changelog = is_partial_update || changelog_layout->need_historical_value; } // rewrite conflict only when partial update or need before @@ -1861,6 +1846,8 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf auto& changelog_ctx = const_cast(transient_changelog_writer->context()); auto& cfg = changelog_ctx.write_changelog_opt().write_changelog_config(); + cfg.layout = changelog_layout; + cfg.write_before = cfg.layout->need_historical_value; cfg.source.tablet_schema = data_ctx.tablet_schema; cfg.source.partial_update_info = data_ctx.partial_update_info; cfg.source.mow_context = data_ctx.mow_context; diff --git a/be/src/storage/tablet/base_tablet.h b/be/src/storage/tablet/base_tablet.h index e570fc79944496..03a82c0c797bb6 100644 --- a/be/src/storage/tablet/base_tablet.h +++ b/be/src/storage/tablet/base_tablet.h @@ -38,6 +38,9 @@ #include "util/bthread_shared_mutex.h" namespace doris { +namespace changelog { +struct ResolvedChangelogLayout; +} struct RowSetSplits; struct RowsetWriterContext; class RowsetWriter; @@ -276,9 +279,10 @@ class BaseTablet : public std::enable_shared_from_this { const Rowset& rowset, std::shared_ptr partial_update_info, int64_t txn_expiration = 0) = 0; - static Status update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInfo* txn_info, - int64_t txn_id, int64_t txn_expiration = 0, - DeleteBitmapPtr tablet_delete_bitmap = nullptr); + static Status update_delete_bitmap( + const BaseTabletSPtr& self, TabletTxnInfo* txn_info, int64_t txn_id, + int64_t txn_expiration = 0, DeleteBitmapPtr tablet_delete_bitmap = nullptr, + std::shared_ptr changelog_layout = nullptr); virtual Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id, DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer, const RowsetIdUnorderedSet& cur_rowset_ids, diff --git a/be/src/storage/tablet/tablet.cpp b/be/src/storage/tablet/tablet.cpp index 68b67c918614e3..7ab8d1fa36d8aa 100644 --- a/be/src/storage/tablet/tablet.cpp +++ b/be/src/storage/tablet/tablet.cpp @@ -2081,11 +2081,6 @@ void Tablet::_init_context_common_fields(RowsetWriterContext& context) { context.enable_unique_key_merge_on_write = enable_unique_key_merge_on_write(); context.encrypt_algorithm = tablet_meta()->encryption_algorithm(); - - if (context.write_changelog_opt().enable) { - context.write_changelog_opt().set_need_before( - tablet_meta()->binlog_config().need_historical_value()); - } } Status Tablet::create_rowset(const RowsetMetaSharedPtr& rowset_meta, RowsetSharedPtr* rowset) { diff --git a/be/src/storage/tablet_info.cpp b/be/src/storage/tablet_info.cpp index b617e7ace6bcc9..fc4529f3f07eae 100644 --- a/be/src/storage/tablet_info.cpp +++ b/be/src/storage/tablet_info.cpp @@ -58,6 +58,7 @@ #include "exprs/function/cast/cast_to_timestamptz.h" #include "exprs/vexpr_context.h" // IWYU pragma: keep #include "exprs/vliteral.h" +#include "storage/changelog.h" namespace doris { @@ -73,6 +74,9 @@ const OlapTableIndexSchema* OlapTableSchemaParam::changelog_index_schema(int64_t void OlapTableIndexSchema::to_protobuf(POlapTableIndexSchema* pindex) const { pindex->set_id(index_id); pindex->set_schema_hash(schema_hash); + if (changelog_column_mappings != nullptr) { + *pindex->mutable_changelog_column_mappings() = *changelog_column_mappings; + } if (changelog_index_id > 0) { pindex->set_changelog_index_id(changelog_index_id); } @@ -195,8 +199,16 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { auto* index = _obj_pool.add(new OlapTableIndexSchema()); index->index_id = p_index.id(); index->schema_hash = p_index.schema_hash(); - if (p_index.has_changelog_index_id()) { + if (p_index.has_changelog_index_id() && p_index.changelog_index_id() > 0) { index->changelog_index_id = p_index.changelog_index_id(); + if (!p_index.has_changelog_column_mappings()) { + return Status::InvalidArgument("source index {} misses changelog mapping", + index->index_id); + } + RETURN_IF_ERROR(changelog::validate_mapping(p_index.changelog_column_mappings())); + index->changelog_column_mappings = + std::make_shared( + p_index.changelog_column_mappings()); } for (const auto& pcolumn_desc : p_index.columns_desc()) { if (_unique_key_update_mode != UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS || @@ -356,8 +368,16 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { auto* index = _obj_pool.add(new OlapTableIndexSchema()); index->index_id = t_index.id; index->schema_hash = t_index.schema_hash; - if (t_index.__isset.changelog_index_id) { + if (t_index.__isset.changelog_index_id && t_index.changelog_index_id > 0) { index->changelog_index_id = t_index.changelog_index_id; + if (!t_index.__isset.changelog_column_mappings) { + return Status::InvalidArgument("source index {} misses changelog mapping", + index->index_id); + } + auto mapping = std::make_shared(); + RETURN_IF_ERROR(changelog::mapping_from_thrift(t_index.changelog_column_mappings, + mapping.get())); + index->changelog_column_mappings = std::move(mapping); } for (const auto& tcolumn_desc : t_index.columns_desc) { if (_unique_key_update_mode != UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS || diff --git a/be/src/storage/tablet_info.h b/be/src/storage/tablet_info.h index 8bddc147f24491..4187efc0da5c77 100644 --- a/be/src/storage/tablet_info.h +++ b/be/src/storage/tablet_info.h @@ -58,6 +58,7 @@ class TupleDescriptor; struct OlapTableIndexSchema { int64_t index_id; int64_t changelog_index_id = 0; + std::shared_ptr changelog_column_mappings; std::vector slots; int32_t schema_hash; std::vector columns; diff --git a/be/src/storage/task/engine_publish_version_task.cpp b/be/src/storage/task/engine_publish_version_task.cpp index 1cea9da30afa44..d75b64f123939b 100644 --- a/be/src/storage/task/engine_publish_version_task.cpp +++ b/be/src/storage/task/engine_publish_version_task.cpp @@ -35,9 +35,11 @@ #include "cloud/config.h" #include "common/logging.h" +#include "storage/changelog.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_meta.h" +#include "storage/tablet/tablet_meta_manager.h" #include "storage/txn/txn_manager.h" #include "storage/utils.h" #include "util/bvar_helper.h" @@ -97,6 +99,18 @@ void EnginePublishVersionTask::add_error_tablet_id(int64_t tablet_id) { Status EnginePublishVersionTask::execute() { Status res = Status::OK(); + std::unordered_map> mappings; + for (const auto& snapshot : _publish_version_req.changelog_column_mappings) { + if (!snapshot.__isset.source_index_id || !snapshot.__isset.mappings) { + return Status::InvalidArgument("publish changelog snapshot misses index/mapping"); + } + auto mapping = std::make_shared(); + RETURN_IF_ERROR(changelog::mapping_from_thrift(snapshot.mappings, mapping.get())); + if (!mappings.emplace(snapshot.source_index_id, std::move(mapping)).second) { + return Status::InvalidArgument("duplicate publish changelog index {}", + snapshot.source_index_id); + } + } int64_t transaction_id = _publish_version_req.transaction_id; OlapStopWatch watch; VLOG_NOTICE << "begin to process publish version. transaction_id=" << transaction_id; @@ -194,6 +208,16 @@ Status EnginePublishVersionTask::execute() { tablet_info.tablet_id); continue; } + auto attach_changelog = txn_info_it->second->attach_changelog; + auto mapping_it = mappings.find(rowset->rowset_meta()->index_id()); + attach_changelog.column_mappings = + mapping_it == mappings.end() ? nullptr : mapping_it->second; + if (attach_changelog.rowset != nullptr && attach_changelog.column_mappings == nullptr) { + add_error_tablet_id(tablet_info.tablet_id); + res = Status::InvalidArgument("publish txn {} tablet {} misses changelog mapping", + transaction_id, tablet_info.tablet_id); + continue; + } // in uniq key model with merge-on-write, we should see all // previous version when update delete bitmap, so add a check // here and wait pre version publish or lock timeout @@ -229,13 +253,15 @@ Status EnginePublishVersionTask::execute() { max_continuous_version.second != max_version) { _handle_publish_version_not_continuous( partition_id, tablet_info, tablet, version, - par_ver_info.commit_tso, max_version, first_time_update, res); + par_ver_info.commit_tso, max_version, first_time_update, res, + attach_changelog.column_mappings); continue; } } else { _handle_publish_version_not_continuous(partition_id, tablet_info, tablet, version, par_ver_info.commit_tso, - max_version, first_time_update, res); + max_version, first_time_update, res, + attach_changelog.column_mappings); continue; } } @@ -250,8 +276,8 @@ Status EnginePublishVersionTask::execute() { } auto tablet_publish_txn_ptr = std::make_shared( - _engine, this, tablet, rowset, txn_info_it->second->attach_changelog, - partition_id, transaction_id, version, tablet_info, par_ver_info.commit_tso); + _engine, this, tablet, rowset, attach_changelog, partition_id, transaction_id, + version, tablet_info, par_ver_info.commit_tso); tablet_tasks.push_back(tablet_publish_txn_ptr); auto submit_st = token->submit_func([=]() { tablet_publish_txn_ptr->handle(); }); #ifndef NDEBUG @@ -332,7 +358,8 @@ Status EnginePublishVersionTask::execute() { void EnginePublishVersionTask::_handle_publish_version_not_continuous( int64_t partition_id, const TabletInfo& tablet_info, const TabletSharedPtr& tablet, const Version& version, const int64_t commit_tso, int64_t max_version, - bool first_time_update, Status& res) { + bool first_time_update, Status& res, + std::shared_ptr column_mappings) { if (config::enable_auto_clone_on_mow_publish_missing_version) { LOG_INFO("mow publish submit missing rowset clone task.") .tag("tablet_id", tablet->tablet_id()) @@ -357,14 +384,20 @@ void EnginePublishVersionTask::_handle_publish_version_not_continuous( // When there are too many missing versions, do not directly retry the // publish and handle it through async publish. if (max_version + config::mow_publish_max_discontinuous_version_num < version.first) { - _engine.add_async_publish_task(partition_id, tablet_info.tablet_id, version.first, - _publish_version_req.transaction_id, false, commit_tso); + auto enqueue_status = _engine.add_async_publish_task( + partition_id, tablet_info.tablet_id, version.first, + _publish_version_req.transaction_id, false, commit_tso, column_mappings); + if (!enqueue_status.ok()) { + res = std::move(enqueue_status); + return; + } } else { _discontinuous_version_tablets->emplace_back( DiscontinuousVersionTablet {.partition_id = partition_id, .tablet_id = tablet_info.tablet_id, .publish_version = version.first, - .commit_tso = commit_tso}); + .commit_tso = commit_tso, + .column_mappings = column_mappings}); } res = Status::Error( "version not continuous for mow, tablet_id={}, " @@ -431,19 +464,19 @@ TabletPublishTxnTask::TabletPublishTxnTask( TabletPublishTxnTask::~TabletPublishTxnTask() = default; -Status publish_version_and_add_rowset(StorageEngine& engine, int64_t partition_id, - const TabletSharedPtr& tablet, const RowsetSharedPtr& rowset, - int64_t transaction_id, const Version& version, - EnginePublishVersionTask* engine_publish_version_task, - TabletPublishStatistics& stats, int64_t commit_tso) { +Status publish_version_and_add_rowset( + StorageEngine& engine, int64_t partition_id, const TabletSharedPtr& tablet, + const RowsetSharedPtr& rowset, int64_t transaction_id, const Version& version, + EnginePublishVersionTask* engine_publish_version_task, TabletPublishStatistics& stats, + int64_t commit_tso, std::shared_ptr column_mappings) { // ATTN: Here, the life cycle needs to be extended to prevent tablet_txn_info.pending_rs_guard in txn // from being released prematurely, causing path gc to mistakenly delete the dat file std::shared_ptr extend_tablet_txn_info_lifetime = nullptr; // Publish the transaction - auto result = - engine.txn_manager()->publish_txn(partition_id, tablet, transaction_id, version, &stats, - extend_tablet_txn_info_lifetime, commit_tso); + auto result = engine.txn_manager()->publish_txn(partition_id, tablet, transaction_id, version, + &stats, extend_tablet_txn_info_lifetime, + commit_tso, std::move(column_mappings)); if (!result.ok()) { LOG(WARNING) << "failed to publish version. rowset_id=" << rowset->rowset_id() << ", tablet_id=" << tablet->tablet_id() << ", txn_id=" << transaction_id @@ -538,9 +571,9 @@ void TabletPublishTxnTask::handle() { rowset_update_lock.lock(); } _stats.schedule_time_us = MonotonicMicros() - _stats.submit_time_us; - _result = publish_version_and_add_rowset(_engine, _partition_id, _tablet, _rowset, - _transaction_id, _version, - _engine_publish_version_task, _stats, _commit_tso); + _result = publish_version_and_add_rowset( + _engine, _partition_id, _tablet, _rowset, _transaction_id, _version, + _engine_publish_version_task, _stats, _commit_tso, _attach_changelog.column_mappings); if (!_result.ok()) { return; @@ -567,11 +600,26 @@ void AsyncTabletPublishTask::handle() { &tablet_related_rs, &tablet_related_txn_infos); auto iter = tablet_related_rs.find(TabletInfo(_tablet->tablet_id(), _tablet->tablet_uid())); if (iter == tablet_related_rs.end()) { + // The transaction was already retired (for example by a normal publish). + // There is no writer context to retry; preserve the existing retirement rule. + static_cast(TabletMetaManager::remove_pending_publish_info( + _tablet->data_dir(), _tablet->tablet_id(), _version)); return; } auto txn_info_it = tablet_related_txn_infos.find(TabletInfo(_tablet->tablet_id(), _tablet->tablet_uid())); DCHECK(txn_info_it != tablet_related_txn_infos.end()); + bool published = false; + // A failed bitmap rewrite must retain both the durable pending record and its + // mapping. Requeue this exact snapshot; successful publish removes it below. + Defer retry([&] { + if (!published) { + CHECK(_engine.add_async_publish_task(_partition_id, _tablet->tablet_id(), _version, + _transaction_id, true, _commit_tso, + _column_mappings) + .ok()); + } + }); RowsetSharedPtr rowset = iter->second; Version version(_version, _version); @@ -600,12 +648,15 @@ void AsyncTabletPublishTask::handle() { auto publish_status = publish_version_and_add_rowset(_engine, _partition_id, _tablet, rowset, _transaction_id, - version, nullptr, _stats, _commit_tso); + version, nullptr, _stats, _commit_tso, _column_mappings); if (!publish_status.ok()) { return; } + published = true; + static_cast(TabletMetaManager::remove_pending_publish_info( + _tablet->data_dir(), _tablet->tablet_id(), _version)); int64_t cost_us = MonotonicMicros() - _stats.submit_time_us; // print stats if publish cost > 500ms g_tablet_publish_latency << cost_us; diff --git a/be/src/storage/task/engine_publish_version_task.h b/be/src/storage/task/engine_publish_version_task.h index afc7b112428a02..eb2c004b85ec7d 100644 --- a/be/src/storage/task/engine_publish_version_task.h +++ b/be/src/storage/task/engine_publish_version_task.h @@ -97,6 +97,7 @@ struct DiscontinuousVersionTablet { int64_t tablet_id; int64_t publish_version; int64_t commit_tso; + std::shared_ptr column_mappings; }; class EnginePublishVersionTask final : public EngineTask { @@ -115,11 +116,11 @@ class EnginePublishVersionTask final : public EngineTask { void add_error_tablet_id(int64_t tablet_id); private: - void _handle_publish_version_not_continuous(int64_t partition_id, const TabletInfo& tablet_info, - const TabletSharedPtr& tablet, - const Version& version, const int64_t commit_tso, - int64_t max_version, bool first_time_update, - Status& res); + void _handle_publish_version_not_continuous( + int64_t partition_id, const TabletInfo& tablet_info, const TabletSharedPtr& tablet, + const Version& version, const int64_t commit_tso, int64_t max_version, + bool first_time_update, Status& res, + std::shared_ptr column_mappings); void _calculate_tbl_num_delta_rows( const std::unordered_map& tablet_id_to_num_delta_rows); @@ -135,8 +136,10 @@ class EnginePublishVersionTask final : public EngineTask { class AsyncTabletPublishTask { public: - AsyncTabletPublishTask(StorageEngine& engine, TabletSharedPtr tablet, int64_t partition_id, - int64_t transaction_id, int64_t version, int64_t commit_tso) + AsyncTabletPublishTask( + StorageEngine& engine, TabletSharedPtr tablet, int64_t partition_id, + int64_t transaction_id, int64_t version, int64_t commit_tso, + std::shared_ptr column_mappings = nullptr) : _engine(engine), _tablet(std::move(tablet)), _partition_id(partition_id), @@ -144,7 +147,8 @@ class AsyncTabletPublishTask { _version(version), _mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, "AsyncTabletPublishTask")), - _commit_tso(commit_tso) { + _commit_tso(commit_tso), + _column_mappings(std::move(column_mappings)) { _stats.submit_time_us = MonotonicMicros(); } ~AsyncTabletPublishTask() = default; @@ -160,6 +164,7 @@ class AsyncTabletPublishTask { TabletPublishStatistics _stats; std::shared_ptr _mem_tracker; int64_t _commit_tso; + std::shared_ptr _column_mappings; }; } // namespace doris diff --git a/be/src/storage/transform/changelog_derive.cpp b/be/src/storage/transform/changelog_derive.cpp index a6bd60dd9f8f91..169197cad752ff 100644 --- a/be/src/storage/transform/changelog_derive.cpp +++ b/be/src/storage/transform/changelog_derive.cpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include "common/cast_set.h" #include "core/assert_cast.h" @@ -27,6 +29,7 @@ #include "core/block/column_with_type_and_name.h" #include "core/column/column_nullable.h" #include "core/column/column_vector.h" +#include "gen_cpp/Descriptors_types.h" #include "storage/changelog.h" #include "storage/key/row_key_encoder.h" #include "storage/mow/historical_row_fetcher.h" @@ -37,12 +40,177 @@ #include "storage/transform/transform_util.h" #include "util/time.h" -namespace doris::segment_v2 { +namespace doris::changelog { + +Status validate_mapping(const PChangelogWriteColumnMappings& mapping) { + if (!mapping.has_need_historical_value()) { + return Status::InvalidArgument("changelog mapping misses need_historical_value"); + } + std::unordered_set source_uids; + std::unordered_set target_uids; + for (const auto& entry : mapping.entries()) { + if (!entry.has_source_column_unique_id() || !entry.has_current_column_unique_id()) { + return Status::InvalidArgument("changelog mapping misses source/current UID"); + } + if (!source_uids.insert(entry.source_column_unique_id()).second || + !target_uids.insert(entry.current_column_unique_id()).second) { + return Status::InvalidArgument("duplicate changelog source/current UID"); + } + if (entry.has_before_column_unique_id() && + (!mapping.need_historical_value() || + !target_uids.insert(entry.before_column_unique_id()).second)) { + return Status::InvalidArgument("invalid or duplicate changelog before UID"); + } + } + return Status::OK(); +} + +Status mapping_from_thrift(const TChangelogWriteColumnMappings& input, + PChangelogWriteColumnMappings* output) { + if (!input.__isset.entries || !input.__isset.need_historical_value) { + return Status::InvalidArgument("changelog mapping misses entries/historical flag"); + } + output->Clear(); + output->set_need_historical_value(input.need_historical_value); + for (const auto& entry : input.entries) { + if (!entry.__isset.source_column_unique_id || !entry.__isset.current_column_unique_id) { + return Status::InvalidArgument("changelog mapping misses source/current UID"); + } + auto* encoded = output->add_entries(); + encoded->set_source_column_unique_id(entry.source_column_unique_id); + encoded->set_current_column_unique_id(entry.current_column_unique_id); + if (entry.__isset.before_column_unique_id) { + encoded->set_before_column_unique_id(entry.before_column_unique_id); + } + } + return validate_mapping(*output); +} namespace { +bool same_type_shape(const TabletColumn& source, const TabletColumn& target, bool top_level) { + if (source.type() != target.type() || source.length() != target.length() || + source.precision() != target.precision() || source.frac() != target.frac() || + source.get_subtype_count() != target.get_subtype_count() || + (source.is_nullable() && !target.is_nullable()) || + (!top_level && source.is_nullable() != target.is_nullable())) { + return false; + } + for (uint32_t i = 0; i < source.get_subtype_count(); ++i) { + if (!same_type_shape(source.get_sub_column(i), target.get_sub_column(i), false)) { + return false; + } + } + return true; +} +} // namespace -// Number of changelog system columns: TSO, LSN, op. -constexpr uint32_t CHANGELOG_COLNUM = 3; +Status resolve_layout(const TabletSchemaSPtr& source, const TabletSchemaSPtr& target, + const PChangelogWriteColumnMappings& mapping, + std::shared_ptr* result) { + RETURN_IF_ERROR(validate_mapping(mapping)); + if (source == nullptr || target == nullptr) { + return Status::InvalidArgument("changelog layout requires source and target schemas"); + } + auto layout = std::make_shared(); + layout->source_schema = source; + layout->target_schema = target; + layout->need_historical_value = mapping.need_historical_value(); + const int tso = target->changelog_tso_col_idx(); + const int lsn = target->changelog_lsn_col_idx(); + const int op = target->changelog_op_col_idx(); + if (tso < 0 || lsn < 0 || op < 0) { + return Status::InvalidArgument("changelog schema misses its hidden columns"); + } + layout->tso_cid = tso; + layout->lsn_cid = lsn; + layout->op_cid = op; + if (target->column(tso).type() != FieldType::OLAP_FIELD_TYPE_BIGINT || + !target->column(tso).is_nullable() || + target->column(lsn).type() != FieldType::OLAP_FIELD_TYPE_BIGINT || + target->column(lsn).is_nullable() || + target->column(op).type() != FieldType::OLAP_FIELD_TYPE_BIGINT) { + return Status::InvalidArgument("invalid changelog system column types"); + } + std::unordered_map source_cids; + std::unordered_map target_cids; + for (uint32_t cid = 0; cid < source->num_columns(); ++cid) { + const auto& column = source->column(cid); + // Hidden non-key execution columns (delete sign, sequence, TSO) have + // no changelog image and are outside the mapping's source namespace. + if (!column.visible() && !column.is_key()) { + continue; + } + if (!source_cids.emplace(column.unique_id(), cid).second) { + return Status::InvalidArgument("duplicate source schema UID"); + } + } + for (uint32_t cid = 0; cid < target->num_columns(); ++cid) { + if (!target_cids.emplace(target->column(cid).unique_id(), cid).second) { + return Status::InvalidArgument("duplicate changelog schema UID"); + } + } + std::unordered_set mapped_source; + std::unordered_set mapped_target = {layout->tso_cid, layout->lsn_cid, layout->op_cid}; + for (const auto& entry : mapping.entries()) { + auto src = source_cids.find(entry.source_column_unique_id()); + auto dst = target_cids.find(entry.current_column_unique_id()); + if (src == source_cids.end() || dst == target_cids.end()) { + return Status::InvalidArgument("unknown changelog source/current UID: {}/{}", + entry.source_column_unique_id(), + entry.current_column_unique_id()); + } + const auto& source_col = source->column(src->second); + const auto& target_col = target->column(dst->second); + if ((!source_col.visible() && !source_col.is_key()) || + source_col.is_key() != target_col.is_key() || + !same_type_shape(source_col, target_col, !source_col.is_key()) || + !mapped_target.insert(dst->second).second) { + return Status::InvalidArgument("invalid changelog current mapping for UID {}", + entry.source_column_unique_id()); + } + mapped_source.insert(src->second); + layout->current_source_cids.push_back(src->second); + layout->current_target_cids.push_back(dst->second); + layout->current_nullable_widening.push_back(!source_col.is_nullable() && + target_col.is_nullable()); + const bool needs_before = mapping.need_historical_value() && !source_col.is_key(); + if (entry.has_before_column_unique_id() != needs_before) { + return Status::InvalidArgument("incomplete changelog before mapping for UID {}", + entry.source_column_unique_id()); + } + if (needs_before) { + auto before = target_cids.find(entry.before_column_unique_id()); + if (before == target_cids.end() || !mapped_target.insert(before->second).second) { + return Status::InvalidArgument("unknown or reused changelog before UID {}", + entry.before_column_unique_id()); + } + const auto& before_col = target->column(before->second); + if (before_col.is_key() || !before_col.is_nullable() || + !same_type_shape(source_col, before_col, true)) { + return Status::InvalidArgument("invalid changelog before type for UID {}", + entry.before_column_unique_id()); + } + layout->before_source_cids.push_back(src->second); + layout->before_target_cids.push_back(before->second); + } + } + for (uint32_t cid = 0; cid < source->num_columns(); ++cid) { + const auto& col = source->column(cid); + if ((col.visible() || col.is_key()) && !mapped_source.contains(cid)) { + return Status::InvalidArgument("unmapped changelog source UID {}", col.unique_id()); + } + } + if (mapped_target.size() != target->num_columns()) { + return Status::InvalidArgument("unmapped changelog target columns"); + } + *result = std::move(layout); + return Status::OK(); +} +} // namespace doris::changelog + +namespace doris::segment_v2 { + +namespace { // Runs the primary-key historical lookup over `block`'s source key (+seq) // columns, leaving the planned reads and the op for each row in `retriever` for @@ -84,17 +252,13 @@ Status setup_retriever_and_lookup(TransformExecContext& ctx, // reads. No-op when the source schema has no value columns. The retriever is // not const: reading BEFORE also loads the old delete signs it keeps. Status fill_before_columns(Block& out, const TabletSchema& changelog_schema, - PrimaryKeyModelRowRetriever* retriever, uint32_t before_col_start, + PrimaryKeyModelRowRetriever* retriever, + const std::vector& before_cids, const std::vector& value_source_cids, size_t num_rows) { size_t value_column_num = value_source_cids.size(); if (value_column_num == 0) { return Status::OK(); } - std::vector before_cids; - for (uint32_t cid = before_col_start; - cid < before_col_start + cast_set(value_column_num); ++cid) { - before_cids.emplace_back(cid); - } Block before_block = changelog_schema.create_storage_block(before_cids); DCHECK(retriever != nullptr); DCHECK_EQ(before_cids.size(), value_source_cids.size()); @@ -161,8 +325,7 @@ void fill_changelog_system_columns(Block& out, const TabletSchema& changelog_sch } } -// Shared setup step: work out the changelog schema layout and consume this -// segment's LSN range (exactly once). +// Borrow the prepared layout and consume this segment's LSN range exactly once. Status resolve_changelog_context(TransformExecContext& ctx, const Block* block, ChangelogDeriveContext* c) { c->changelog_schema = ctx.tablet_schema; @@ -178,6 +341,9 @@ Status resolve_changelog_context(TransformExecContext& ctx, const Block* block, "changelog derive needs the source tablet to look history up, tablet_id={}", ctx.rowset_ctx->tablet_id); } + if (cfg.layout == nullptr) { + return Status::InvalidArgument("changelog derive requires a resolved column mapping"); + } c->num_rows = block->rows(); // the LSN range for each row was registered per segment id before this @@ -190,52 +356,13 @@ Status resolve_changelog_context(TransformExecContext& ctx, const Block* block, ctx.rowset_ctx->remove_segment_allocated_lsns(ctx.segment_id); CHECK(c->lsn_ids->size() >= c->num_rows) << c->lsn_ids->size() << " vs " << c->num_rows; - // Preserve the source writer's layout: system columns may be a prefix or - // suffix, and source hidden non-key columns are omitted while hidden keys - // remain part of the normal row image. - int tso_col_id = c->changelog_schema->changelog_tso_col_idx(); - int lsn_col_id = c->changelog_schema->changelog_lsn_col_idx(); - int op_col_id = c->changelog_schema->changelog_op_col_idx(); - // The three positions are derived from the column names, so a schema whose hidden columns - // carry other names has none of them. Example: a load into a changelog tablet created by a - // build that named them differently. Fail that load instead of aborting the BE. - if (tso_col_id < 0 || lsn_col_id < 0 || op_col_id < 0) { - return Status::InternalError( - "changelog schema misses its hidden columns: {}={}, {}={}, {}={}, tablet_id={}", - CHANGELOG_TSO_COL, tso_col_id, CHANGELOG_LSN_COL, lsn_col_id, CHANGELOG_OP_COL, - op_col_id, ctx.rowset_ctx->tablet_id); - } - c->changelog_tso_cid = cast_set(tso_col_id); - c->changelog_lsn_cid = cast_set(lsn_col_id); - c->changelog_op_cid = cast_set(op_col_id); - c->normal_col_start = tso_col_id == 0 ? CHANGELOG_COLNUM : 0; - for (uint32_t cid = 0; cid < c->source_schema->num_columns(); ++cid) { - const auto& column = c->source_schema->column(cid); - if (column.visible() || column.is_key()) { - c->normal_source_cids.emplace_back(cid); - } - if (column.visible() && !column.is_key()) { - c->value_source_cids.emplace_back(cid); - } - } - c->before_col_start = c->normal_col_start + cast_set(c->normal_source_cids.size()); - c->write_before = cfg.write_before; - // The schema must carry exactly the BEFORE columns this flush fills: one per - // source value column when write_before is on, none otherwise. Too few and - // the BEFORE fill would write over the TSO/LSN columns; too many and those - // columns would stay empty in an otherwise full block, leaving the writer a - // block whose columns hold different row counts. - const size_t expected_before_columns = c->write_before ? c->value_source_cids.size() : 0; - const size_t expected_columns = - c->normal_source_cids.size() + expected_before_columns + CHANGELOG_COLNUM; - if (c->changelog_schema->num_columns() != expected_columns) { - return Status::InternalError( - "changelog schema does not match the derived block: num_columns={}, expected={} " - "(normal={}, before={}, system={}), write_before={}, tablet_id={}", - c->changelog_schema->num_columns(), expected_columns, c->normal_source_cids.size(), - expected_before_columns, CHANGELOG_COLNUM, c->write_before, - ctx.rowset_ctx->tablet_id); - } + c->layout = cfg.layout.get(); + c->changelog_schema = c->layout->target_schema; + c->source_schema = c->layout->source_schema; + c->changelog_tso_cid = c->layout->tso_cid; + c->changelog_lsn_cid = c->layout->lsn_cid; + c->changelog_op_cid = c->layout->op_cid; + c->write_before = c->layout->need_historical_value; return Status::OK(); } @@ -268,21 +395,22 @@ Status emit_changelog_block(const ChangelogDeriveContext& c, const Block* after_ c.changelog_schema->num_columns()); } // key + AFTER columns: COW pointers of the source-layout visible columns - for (size_t ordinal = 0; ordinal < c.normal_source_cids.size(); ++ordinal) { - const uint32_t source_cid = c.normal_source_cids[ordinal]; + for (size_t ordinal = 0; ordinal < c.layout->current_source_cids.size(); ++ordinal) { + const uint32_t source_cid = c.layout->current_source_cids[ordinal]; ColumnPtr after_column = after_src->get_by_position(source_cid).column; // The changelog schema declares every AFTER value column nullable and `out` keeps that // type when a column is replaced, so a NOT NULL source column is wrapped to match it; a // nullable ARRAY, MAP or STRUCT writer would refuse it otherwise. Keys keep their own // nullability. - if (!c.source_schema->column(source_cid).is_key()) { + if (c.layout->current_nullable_widening[ordinal]) { after_column = make_nullable(after_column); } - out.replace_by_position(c.normal_col_start + ordinal, std::move(after_column)); + out.replace_by_position(c.layout->current_target_cids[ordinal], std::move(after_column)); } if (c.write_before) { - RETURN_IF_ERROR(fill_before_columns(out, *c.changelog_schema, retriever, c.before_col_start, - c.value_source_cids, c.num_rows)); + RETURN_IF_ERROR(fill_before_columns(out, *c.changelog_schema, retriever, + c.layout->before_target_cids, + c.layout->before_source_cids, c.num_rows)); } // An insert whose old row is a tombstone is an append, not an update. Run // this after the BEFORE read, which already loaded the old delete signs. @@ -310,7 +438,7 @@ bool changelog_needs_historical_lookup(const RowsetWriterContext& context) { const bool is_partial_update = cfg.source.partial_update_info != nullptr && cfg.source.partial_update_info->is_partial_update() && is_source_direct_write; - return is_partial_update || cfg.write_before; + return is_partial_update || (cfg.layout != nullptr && cfg.layout->need_historical_value); } Status ChangelogDeriveStage::apply(TransformExecContext& ctx, Block* block) const { diff --git a/be/src/storage/transform/changelog_derive.h b/be/src/storage/transform/changelog_derive.h index db12888427f51e..eda2dfcd15f208 100644 --- a/be/src/storage/transform/changelog_derive.h +++ b/be/src/storage/transform/changelog_derive.h @@ -22,6 +22,9 @@ namespace doris { struct RowsetWriterContext; +namespace changelog { +struct ResolvedChangelogLayout; +} namespace segment_v2 { @@ -37,7 +40,7 @@ bool changelog_needs_historical_lookup(const RowsetWriterContext& context); // Context for each flush that the base stage works out once and passes to // derive(): the changelog/source schemas, the consumed LSN range, row count, and -// column layout [keys..., AFTER..., (BEFORE...), TSO, LSN, OP]. +// explicit source/target CID layout resolved when the writer was prepared. struct ChangelogDeriveContext { TabletSchemaSPtr changelog_schema; TabletSchemaSPtr source_schema; @@ -46,10 +49,7 @@ struct ChangelogDeriveContext { uint32_t changelog_tso_cid = 0; uint32_t changelog_lsn_cid = 0; uint32_t changelog_op_cid = 0; - uint32_t normal_col_start = 0; - uint32_t before_col_start = 0; - std::vector normal_source_cids; - std::vector value_source_cids; + const changelog::ResolvedChangelogLayout* layout = nullptr; bool write_before = false; }; diff --git a/be/src/storage/txn/txn_manager.cpp b/be/src/storage/txn/txn_manager.cpp index b12b5018177d3d..d1b56fcddc7dc6 100644 --- a/be/src/storage/txn/txn_manager.cpp +++ b/be/src/storage/txn/txn_manager.cpp @@ -241,14 +241,14 @@ Status TxnManager::commit_txn(TPartitionId partition_id, const Tablet& tablet, std::move(guard), is_recovery, partial_update_info, attach_changelog); } -Status TxnManager::publish_txn(TPartitionId partition_id, const TabletSharedPtr& tablet, - TTransactionId transaction_id, const Version& version, - TabletPublishStatistics* stats, - std::shared_ptr& extend_tablet_txn_info, - const int64_t commit_tso) { +Status TxnManager::publish_txn( + TPartitionId partition_id, const TabletSharedPtr& tablet, TTransactionId transaction_id, + const Version& version, TabletPublishStatistics* stats, + std::shared_ptr& extend_tablet_txn_info, const int64_t commit_tso, + std::shared_ptr column_mappings) { return publish_txn(tablet->data_dir()->get_meta(), partition_id, transaction_id, tablet->tablet_id(), tablet->tablet_uid(), version, stats, - extend_tablet_txn_info, commit_tso); + extend_tablet_txn_info, commit_tso, std::move(column_mappings)); } void TxnManager::abort_txn(TPartitionId partition_id, TTransactionId transaction_id, @@ -532,12 +532,12 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id, } // remove a txn from txn manager -Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, - TTransactionId transaction_id, TTabletId tablet_id, - TabletUid tablet_uid, const Version& version, - TabletPublishStatistics* stats, - std::shared_ptr& extend_tablet_txn_info, - const int64_t commit_tso) { +Status TxnManager::publish_txn( + OlapMeta* meta, TPartitionId partition_id, TTransactionId transaction_id, + TTabletId tablet_id, TabletUid tablet_uid, const Version& version, + TabletPublishStatistics* stats, std::shared_ptr& extend_tablet_txn_info, + const int64_t commit_tso, + std::shared_ptr column_mappings) { auto tablet = _engine.tablet_manager()->get_tablet(tablet_id); if (tablet == nullptr) { return Status::OK(); @@ -575,6 +575,19 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, partition_id, transaction_id, tablet_info.to_string(), commit_tso); } const auto& attached_changelog = tablet_txn_info->attach_changelog.rowset; + std::shared_ptr changelog_layout; + if (attached_changelog != nullptr) { + // Recovery has the rowsets but no in-memory writer. The transaction's FE + // snapshot must arrive with publish, before either rowset becomes visible. + if (column_mappings == nullptr) { + return Status::InvalidArgument("publish txn {} tablet {} misses changelog mappings", + transaction_id, tablet_id); + } + RETURN_IF_ERROR(changelog::resolve_layout( + rowset->tablet_schema()->copy_without_variant_extracted_columns(), + attached_changelog->tablet_schema()->copy_without_variant_extracted_columns(), + *column_mappings, &changelog_layout)); + } const bool requires_commit_tso = rowset->tablet_schema()->is_tso_enabled() || (attached_changelog != nullptr && attached_changelog->tablet_schema()->is_tso_enabled()); @@ -648,8 +661,8 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, tablet_txn_info->delete_bitmap)); } - RETURN_IF_ERROR( - Tablet::update_delete_bitmap(tablet, tablet_txn_info.get(), transaction_id)); + RETURN_IF_ERROR(Tablet::update_delete_bitmap(tablet, tablet_txn_info.get(), transaction_id, + 0, nullptr, changelog_layout)); int64_t t3 = MonotonicMicros(); stats->calc_delete_bitmap_time_us = t3 - t2; RETURN_IF_ERROR(TabletMetaManager::save_delete_bitmap( diff --git a/be/src/storage/txn/txn_manager.h b/be/src/storage/txn/txn_manager.h index c6c7c2c6ffebbc..256c9edc440628 100644 --- a/be/src/storage/txn/txn_manager.h +++ b/be/src/storage/txn/txn_manager.h @@ -49,6 +49,7 @@ namespace doris { class OlapMeta; +class PChangelogWriteColumnMappings; struct TabletPublishStatistics; struct PartialUpdateInfo; @@ -64,6 +65,7 @@ enum class PublishStatus { INIT = 0, PREPARE = 1, SUCCEED = 2 }; // The changelog rowset and its independent changelog tablet, carried through commit and publish. struct ChangelogTxnInfo { + std::shared_ptr column_mappings; RowsetSharedPtr rowset; BaseTabletSPtr tablet; // Delete bitmap deltas that should be applied to the independent changelog tablet. @@ -184,11 +186,11 @@ class TxnManager { std::shared_ptr partial_update_info = nullptr, const ChangelogTxnInfo& attach_changelog = {}); - Status publish_txn(TPartitionId partition_id, const TabletSharedPtr& tablet, - TTransactionId transaction_id, const Version& version, - TabletPublishStatistics* stats, - std::shared_ptr& extend_tablet_txn_info, - const int64_t commit_tso = -1); + Status publish_txn( + TPartitionId partition_id, const TabletSharedPtr& tablet, TTransactionId transaction_id, + const Version& version, TabletPublishStatistics* stats, + std::shared_ptr& extend_tablet_txn_info, const int64_t commit_tso = -1, + std::shared_ptr column_mappings = nullptr); // delete the txn from manager if it is not committed(not have a valid rowset) Status rollback_txn(TPartitionId partition_id, const Tablet& tablet, @@ -205,11 +207,12 @@ class TxnManager { // remove a txn from txn manager // not persist rowset meta because - Status publish_txn(OlapMeta* meta, TPartitionId partition_id, TTransactionId transaction_id, - TTabletId tablet_id, TabletUid tablet_uid, const Version& version, - TabletPublishStatistics* stats, - std::shared_ptr& extend_tablet_txn_info, - const int64_t commit_tso = -1); + Status publish_txn( + OlapMeta* meta, TPartitionId partition_id, TTransactionId transaction_id, + TTabletId tablet_id, TabletUid tablet_uid, const Version& version, + TabletPublishStatistics* stats, std::shared_ptr& extend_tablet_txn_info, + const int64_t commit_tso = -1, + std::shared_ptr column_mappings = nullptr); // only abort not committed txn void abort_txn(TPartitionId partition_id, TTransactionId transaction_id, TTabletId tablet_id, diff --git a/be/test/exec/scan/olap_scanner_test.cpp b/be/test/exec/scan/olap_scanner_test.cpp index a3f20ef6b5710e..019581b7ae839b 100644 --- a/be/test/exec/scan/olap_scanner_test.cpp +++ b/be/test/exec/scan/olap_scanner_test.cpp @@ -23,7 +23,11 @@ #include "cloud/config.h" #include "common/config.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" #include "io/io_common.h" +#include "runtime/descriptors.h" +#include "storage/schema.h" #include "testutil/mock/mock_runtime_state.h" namespace doris { @@ -104,4 +108,175 @@ TEST(OlapScannerTest, BuildScoreRuntimeCollectionIoContextAllowsMissingQueryCont EXPECT_EQ(io_ctx.remote_scan_cache_write_limiter, nullptr); } +namespace { + +struct ChangelogReadFixture { + TabletSchema tablet; + std::vector> owned_slots; + std::vector slots; + std::unique_ptr read_schema; + TOlapScanNode scan_node; + + explicit ChangelogReadFixture(bool historical = true) { + for (const auto& [uid, name] : {std::pair {10, "key"}, + {11, "current"}, + {12, "unrelated_before_name"}, + {13, CHANGELOG_TSO_COL}, + {14, CHANGELOG_OP_COL}}) { + if (!historical && uid == 12) { + continue; + } + TabletColumn column; + column.set_unique_id(uid); + column.set_name(name); + column.set_is_key(uid == 10); + column.set_type(FieldType::OLAP_FIELD_TYPE_BIGINT); + tablet.append_column(column); + } + // The scan tuple order and IDs deliberately differ from both UID and tablet ordinal. + std::vector columns; + for (int uid : {14, 12, 10, 13, 11}) { + if (!historical && uid == 12) { + continue; + } + auto column = tablet.columns()[tablet.field_index(uid)]; + TSlotDescriptor slot; + slot.__set_id(100 - uid); + slot.__set_col_unique_id(uid); + slot.__set_colName("slot_alias_" + std::to_string(uid)); + slot.__set_nullIndicatorBit(-1); + column->get_vec_type()->to_thrift(slot.slotType); + owned_slots.push_back(std::make_unique(slot)); + slots.push_back(owned_slots.back().get()); + columns.push_back(column); + } + read_schema = std::make_unique(std::move(columns)); + TChangelogReadColumnMappings mappings; + std::vector pairs; + if (historical) { + TChangelogReadColumnPair pair; + pair.__set_current_slot_id(89); + pair.__set_before_slot_id(88); + pairs.push_back(pair); + } + mappings.__set_pairs(pairs); + scan_node.__set_changelog_column_mappings(mappings); + } + + Status init(TChangelogScanType::type mode) { + return OlapScanner::_init_changelog_read_mappings(scan_node, slots, tablet, mode, + *read_schema); + } +}; + +} // namespace + +TEST(OlapScannerTest, ExplicitChangelogPairsResolveFinalTupleSlots) { + ChangelogReadFixture fixture; + ASSERT_TRUE(fixture.init(TChangelogScanType::MIN_DELTA).ok()); + EXPECT_EQ(1, fixture.read_schema->before_column_ordinal(4)); + EXPECT_EQ(3, fixture.read_schema->tso_ordinal()); + EXPECT_EQ(0, fixture.read_schema->op_ordinal()); + EXPECT_EQ(-1, fixture.read_schema->lsn_ordinal()); +} + +TEST(OlapScannerTest, ChangelogMappingsDistinguishMissingAndEmpty) { + ChangelogReadFixture fixture(false); + EXPECT_TRUE(fixture.init(TChangelogScanType::DETAIL).ok()); + EXPECT_TRUE(fixture.read_schema->changelog_value_column_pairs().empty()); + fixture.scan_node.changelog_column_mappings.__isset.pairs = false; + EXPECT_FALSE(fixture.init(TChangelogScanType::DETAIL).ok()); + fixture.scan_node.__isset.changelog_column_mappings = false; + EXPECT_FALSE(fixture.init(TChangelogScanType::MIN_DELTA).ok()); + EXPECT_TRUE(fixture.init(TChangelogScanType::APPEND_ONLY).ok()); +} + +TEST(OlapScannerTest, ChangelogMappingsRejectUnknownAndIncompletePairs) { + ChangelogReadFixture fixture; + auto& pair = fixture.scan_node.changelog_column_mappings.pairs[0]; + pair.__set_before_slot_id(999); + EXPECT_FALSE(fixture.init(TChangelogScanType::DETAIL).ok()); + pair.__set_before_slot_id(88); + pair.__isset.current_slot_id = false; + EXPECT_FALSE(fixture.init(TChangelogScanType::DETAIL).ok()); + pair.__set_current_slot_id(89); + pair.__set_before_slot_id(90); // key column + EXPECT_FALSE(fixture.init(TChangelogScanType::DETAIL).ok()); + fixture.scan_node.changelog_column_mappings.pairs.clear(); + EXPECT_FALSE(fixture.init(TChangelogScanType::MIN_DELTA).ok()); +} + +TEST(OlapScannerTest, MinDeltaRequiresUnprojectedValues) { + ChangelogReadFixture fixture; + TabletColumn missing; + missing.set_unique_id(15); + missing.set_name("unprojected_value"); + missing.set_type(FieldType::OLAP_FIELD_TYPE_BIGINT); + fixture.tablet.append_column(missing); + EXPECT_FALSE(fixture.init(TChangelogScanType::MIN_DELTA).ok()); + EXPECT_TRUE(fixture.init(TChangelogScanType::DETAIL).ok()); +} + +TEST(OlapScannerTest, AppendOnlyAllowsNestedPruningButMinDeltaRequiresFullShape) { + ChangelogReadFixture fixture; + for (int uid : {11, 12}) { + auto column = fixture.tablet.columns()[fixture.tablet.field_index(uid)]; + column->set_type(FieldType::OLAP_FIELD_TYPE_STRUCT); + for (const std::string& name : {"a", "b"}) { + TabletColumn child; + child.set_name(name); + child.set_type(FieldType::OLAP_FIELD_TYPE_INT); + column->add_sub_column(child); + } + } + DataTypes types; + for (const auto& column : fixture.read_schema->columns()) { + if (column->unique_id() == 11 || column->unique_id() == 12) { + types.push_back(std::make_shared( + DataTypes {std::make_shared()}, Strings {"a"})); + } else { + types.push_back(column->get_vec_type()); + } + } + fixture.read_schema = + std::make_unique(fixture.read_schema->columns(), std::move(types)); + EXPECT_FALSE(fixture.init(TChangelogScanType::MIN_DELTA).ok()); + fixture.scan_node.changelog_column_mappings.pairs.clear(); + EXPECT_TRUE(fixture.init(TChangelogScanType::APPEND_ONLY).ok()); + auto pruned = std::make_shared(DataTypes {std::make_shared()}, + Strings {"a"}); + EXPECT_TRUE(fixture.read_schema->data_type(4)->equals(*pruned)); +} + +TEST(OlapScannerTest, ChangelogScanRequiresSystemColumns) { + ChangelogReadFixture fixture; + fixture.slots.erase(fixture.slots.begin()); // omit OP from the final tuple + auto columns = fixture.read_schema->columns(); + columns.erase(columns.begin()); + fixture.read_schema = std::make_unique(std::move(columns)); + EXPECT_FALSE(fixture.init(TChangelogScanType::DETAIL).ok()); +} + +TEST(OlapScannerTest, KeyOnlyMinDeltaAcceptsExplicitEmptyMappings) { + ChangelogReadFixture fixture; + std::vector columns; + std::vector slots; + for (size_t ordinal = 0; ordinal < fixture.slots.size(); ++ordinal) { + auto column = fixture.read_schema->columns()[ordinal]; + if (column->unique_id() == 11 || column->unique_id() == 12) { + continue; + } + columns.push_back(column); + slots.push_back(fixture.slots[ordinal]); + } + fixture.tablet.clear_columns(); + for (const auto& column : columns) { + fixture.tablet.append_column(*column); + } + fixture.read_schema = std::make_unique(std::move(columns)); + fixture.slots = std::move(slots); + fixture.scan_node.changelog_column_mappings.pairs.clear(); + EXPECT_TRUE(fixture.init(TChangelogScanType::MIN_DELTA).ok()); +} + } // namespace doris diff --git a/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp b/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp index e53fcadff11ffb..89ff08a9ab464c 100644 --- a/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp +++ b/be/test/olap/rowset/cloud_group_rowset_builder_writer_test.cpp @@ -30,6 +30,7 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/cloud_txn_delete_bitmap_cache.h" #include "common/cast_set.h" #include "common/config.h" #include "core/block/block.h" @@ -294,6 +295,7 @@ class CloudGroupRowsetBuilderWriterTest : public testing::Test { cfg.source.tablet_schema = _tablet->tablet_schema(); cfg.source.base_tablet = _tablet; cfg.source.source_write_type = DataWriteType::TYPE_DIRECT; + RETURN_IF_ERROR(testutil::resolve_test_changelog_layout(changelog_context)); auto lsn_buffer = AutoIncIDBuffer::create_shared(1, 1, kChangelogLsnAutoIncId); lsn_buffer->append_range_for_test(1000, num_rows); std::vector allocated_lsns; @@ -379,4 +381,40 @@ TEST_F(CloudGroupRowsetBuilderWriterTest, writerBuildsChangelogMeta) { assert_rowset_meta(rowsets[0], rowsets[1]); } +TEST_F(CloudGroupRowsetBuilderWriterTest, MappingSurvivesBitmapEviction) { + CloudTxnDeleteBitmapCache cache(1024 * 1024); + ASSERT_TRUE(cache.init().ok()); + ChangelogTxnInfo attached; + attached.column_mappings = std::make_shared( + testutil::create_changelog_mapping(*_tablet->tablet_schema(), + *_changelog_tablet->tablet_schema(), false)); + const auto snapshot = attached.column_mappings; + constexpr int64_t txn_id = 991; + const auto tablet_id = _tablet->tablet_id(); + cache.set_tablet_txn_info(txn_id, tablet_id, std::make_shared(tablet_id), {}, + nullptr, INT64_MAX, nullptr, attached); + attached = {}; + cache.erase(CacheKey(fmt::format("{}/{}", txn_id, tablet_id))); + RowsetSharedPtr rowset; + DeleteBitmapPtr bitmap; + RowsetIdUnorderedSet rowset_ids; + int64_t expiration; + std::shared_ptr partial_update; + std::shared_ptr publish_status; + TxnPublishInfo publish_info; + ChangelogTxnInfo recovered; + ASSERT_TRUE(cache.get_tablet_txn_info(txn_id, tablet_id, &rowset, &bitmap, &rowset_ids, + &expiration, &partial_update, &publish_status, + &publish_info, &recovered) + .ok()); + EXPECT_EQ(recovered.column_mappings, snapshot); + EXPECT_EQ(*publish_status, PublishStatus::INIT); + EXPECT_TRUE(bitmap != nullptr); + cache.remove_unused_tablet_txn_info(txn_id, tablet_id); + EXPECT_TRUE(cache.get_tablet_txn_info(txn_id, tablet_id, &rowset, &bitmap, &rowset_ids, + &expiration, &partial_update, &publish_status, + &publish_info, &recovered) + .is()); +} + } // namespace doris diff --git a/be/test/olap/rowset/group_rowset_builder_test.cpp b/be/test/olap/rowset/group_rowset_builder_test.cpp index a7625990c053f6..837d96db730d91 100644 --- a/be/test/olap/rowset/group_rowset_builder_test.cpp +++ b/be/test/olap/rowset/group_rowset_builder_test.cpp @@ -280,4 +280,38 @@ TEST_F(GroupRowsetBuilderTest, recoverMultipleChangelogPairsInOneTxn) { } } +TEST_F(GroupRowsetBuilderTest, MappingRequiredOnlyForPositiveSourceRelationship) { + const auto descriptors = testutil::create_descriptor_table({}); + TOlapTableSchemaParam wire; + wire.db_id = 1; + wire.table_id = 2; + wire.version = 1; + wire.tuple_desc = descriptors.tupleDescriptors[0]; + wire.indexes.resize(1); + wire.indexes[0].id = 10; + wire.indexes[0].schema_hash = 0; + // FE sets this field on ordinary indexes too; explicit zero is ordinary. + wire.indexes[0].__set_changelog_index_id(0); + TOlapTableIndexSchema target; + target.id = 20; + target.schema_hash = 0; + target.__set_changelog_index_id(20); + wire.__set_changelog_index_schemas({target}); + OlapTableSchemaParam ordinary; + ASSERT_TRUE(ordinary.init(wire).ok()); + POlapTableSchemaParam encoded; + ordinary.to_protobuf(&encoded); + encoded.mutable_indexes(0)->set_changelog_index_id(0); + encoded.mutable_changelog_index_schemas(0)->set_changelog_index_id(20); + OlapTableSchemaParam recovered; + EXPECT_TRUE(recovered.init(encoded).ok()); + + wire.indexes[0].__set_changelog_index_id(20); + OlapTableSchemaParam missing_thrift; + EXPECT_FALSE(missing_thrift.init(wire).ok()); + encoded.mutable_indexes(0)->set_changelog_index_id(20); + OlapTableSchemaParam missing_protobuf; + EXPECT_FALSE(missing_protobuf.init(encoded).ok()); +} + } // namespace doris diff --git a/be/test/olap/rowset/group_rowset_writer_test.cpp b/be/test/olap/rowset/group_rowset_writer_test.cpp index fb30506633454f..30776fe784c041 100644 --- a/be/test/olap/rowset/group_rowset_writer_test.cpp +++ b/be/test/olap/rowset/group_rowset_writer_test.cpp @@ -182,6 +182,7 @@ class GroupRowsetWriterTest : public testing::Test { changelog_options.source.partial_update_info = partial_update_info; changelog_options.source.mow_context = mow_context; changelog_options.source.source_write_type = DataWriteType::TYPE_DIRECT; + RETURN_IF_ERROR_RESULT(testutil::resolve_test_changelog_layout(changelog_context)); auto lsn_buffer = AutoIncIDBuffer::create_shared(1, 1, kChangelogLsnAutoIncId); lsn_buffer->append_range_for_test(1000, num_rows); @@ -260,6 +261,7 @@ class GroupRowsetWriterTest : public testing::Test { cfg.source.base_tablet = _tablet; cfg.source.is_transient_rowset_writer = true; cfg.source.source_write_type = DataWriteType::TYPE_DIRECT; + RETURN_IF_ERROR(testutil::resolve_test_changelog_layout(changelog_context)); auto lsn_buffer = AutoIncIDBuffer::create_shared(1, 1, kChangelogLsnAutoIncId); lsn_buffer->append_range_for_test(1000, num_rows); std::vector allocated_lsns; @@ -407,8 +409,7 @@ TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*changelog_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr rowset_reader; @@ -479,8 +480,7 @@ TEST_F(GroupRowsetWriterTest, keyOnlyFixedPartialUpdatePreservesNarrowBlock) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*changelog_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr rowset_reader; diff --git a/be/test/storage/changelog_vmerge_compaction_test.cpp b/be/test/storage/changelog_vmerge_compaction_test.cpp index 606e5b9cacba1d..a63473fdefc529 100644 --- a/be/test/storage/changelog_vmerge_compaction_test.cpp +++ b/be/test/storage/changelog_vmerge_compaction_test.cpp @@ -273,8 +273,7 @@ class ChangelogVmergeCompactionTest : public testing::Test { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr reader; diff --git a/be/test/storage/compaction/ordered_data_compaction_test.cpp b/be/test/storage/compaction/ordered_data_compaction_test.cpp index 39d7911244c0d7..f8a669c2ae3ba1 100644 --- a/be/test/storage/compaction/ordered_data_compaction_test.cpp +++ b/be/test/storage/compaction/ordered_data_compaction_test.cpp @@ -515,8 +515,7 @@ TEST_F(OrderedDataCompactionTest, test_01) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; diff --git a/be/test/storage/compaction/segcompaction_mow_test.cpp b/be/test/storage/compaction/segcompaction_mow_test.cpp index 3e6f9207096090..19e1e5713de0fc 100644 --- a/be/test/storage/compaction/segcompaction_mow_test.cpp +++ b/be/test/storage/compaction/segcompaction_mow_test.cpp @@ -250,8 +250,7 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); reader_context.stats = &_stats; reader_context.delete_bitmap = delete_bitmap; diff --git a/be/test/storage/compaction/segcompaction_test.cpp b/be/test/storage/compaction/segcompaction_test.cpp index c9f86475da2795..e7515c4de605e5 100644 --- a/be/test/storage/compaction/segcompaction_test.cpp +++ b/be/test/storage/compaction/segcompaction_test.cpp @@ -406,8 +406,7 @@ class SegCompactionTest : public testing::Test { reader_context.read_schema = read_schema; ASSERT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); reader_context.stats = &_stats; @@ -559,8 +558,7 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); reader_context.stats = &_stats; @@ -1071,8 +1069,7 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); reader_context.stats = &_stats; reader_context.is_unique = true; @@ -1343,8 +1340,7 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); reader_context.stats = &_stats; // reader_context.is_unique = true; diff --git a/be/test/storage/compaction/vertical_compaction_test.cpp b/be/test/storage/compaction/vertical_compaction_test.cpp index b2c26c37019333..ee7aedf91f5bdc 100644 --- a/be/test/storage/compaction/vertical_compaction_test.cpp +++ b/be/test/storage/compaction/vertical_compaction_test.cpp @@ -646,8 +646,7 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMerge) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; @@ -788,8 +787,7 @@ TEST_F(VerticalCompactionTest, MergeHonorsKeyRanges) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_reader); @@ -909,8 +907,7 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMerge) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; @@ -1020,8 +1017,7 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMerge) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; @@ -1285,8 +1281,7 @@ TEST_F(VerticalCompactionTest, TestUniqueKeySegmentContextMemoryAmplification) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_rs_reader); @@ -1408,8 +1403,7 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMergeWithDelete) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; @@ -1514,8 +1508,7 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMergeWithDelete) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; @@ -1609,8 +1602,7 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge) { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; LOG(INFO) << "create rowset reader in test"; @@ -1808,8 +1800,7 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMergeWithNullableSparseColum reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); diff --git a/be/test/storage/iterator/block_reader_change_next_block_test.cpp b/be/test/storage/iterator/block_reader_change_next_block_test.cpp index 579126db74c8c3..1bd6c04bb41eaf 100644 --- a/be/test/storage/iterator/block_reader_change_next_block_test.cpp +++ b/be/test/storage/iterator/block_reader_change_next_block_test.cpp @@ -354,12 +354,13 @@ ReadSchemaSPtr make_read_schema(const std::vector& names = { // Wire a BlockReader as if init() had already completed for a changelog change // scan over the fixed 6-column schema, then plug in the fake merge iterator. void configure_reader(BlockReader& reader, std::shared_ptr source, size_t batch_size, - TabletSchemaSPtr schema = make_test_tablet_schema()) { + TabletSchemaSPtr schema = make_test_tablet_schema(), + ReadSchema::ChangelogValueColumnPairs pairs = {{1, 2}}) { config::enable_adaptive_batch_size = false; reader._reader_context.batch_size = batch_size; - // The physical tablet schema supplies stable unique ids for AFTER/BEFORE pairing. The fake - // source block supplies the materialized types used by this read schema. + // The fixture supplies explicit pairs as Scanner does. The fake source block supplies + // materialized types, independently of the names stored in the tablet schema. reader._tablet_schema = std::move(schema); ASSERT_EQ(reader._tablet_schema->num_columns(), source->columns()); std::vector read_types; @@ -371,9 +372,9 @@ void configure_reader(BlockReader& reader, std::shared_ptr source, size_t std::make_shared(reader._tablet_schema->columns(), std::move(read_types)); EXPECT_TRUE(read_schema ->init_from_tablet_schema(*reader._tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/true) + /*merge_by_sequence_mapping=*/false) .ok()); + ASSERT_TRUE(read_schema->init_changelog_column_mappings(*reader._tablet_schema, pairs).ok()); reader._read_schema = std::move(read_schema); reader._next_row.block = source; @@ -457,6 +458,7 @@ class BlockReaderChangeNextBlockTest : public testing::Test { TEST_F(BlockReaderChangeNextBlockTest, ChangelogSchemaWithoutBeforeUsesCurrentColumn) { auto read_schema = make_read_schema({"key", "val", CHANGELOG_TSO_COL, CHANGELOG_OP_COL}); + ASSERT_TRUE(read_schema->init_changelog_column_mappings({}, 2, -1, 3).ok()); EXPECT_EQ(VAL_IDX, read_schema->before_column_ordinal(VAL_IDX)); } @@ -464,6 +466,7 @@ TEST_F(BlockReaderChangeNextBlockTest, ChangelogSchemaDoesNotRequireLsn) { auto read_schema = make_read_schema({"key", "val", changelog::build_before_column_name("val"), CHANGELOG_TSO_COL, CHANGELOG_OP_COL}); + ASSERT_TRUE(read_schema->init_changelog_column_mappings({{1, 2}}, 3, -1, 4).ok()); EXPECT_EQ(-1, read_schema->lsn_ordinal()); EXPECT_EQ(2, read_schema->before_column_ordinal(VAL_IDX)); } @@ -580,7 +583,7 @@ TEST_F(BlockReaderChangeNextBlockTest, MinDeltaNoOpWithCollidingBeforeNameIsSkip {{"v", FieldType::OLAP_FIELD_TYPE_BIGINT}, {changelog::build_before_column_name("v"), FieldType::OLAP_FIELD_TYPE_BIGINT}}); BlockReader reader; - configure_reader(reader, source, 16, std::move(schema)); + configure_reader(reader, source, 16, std::move(schema), {{1, 3}, {2, 4}}); bool eof = false; Block output = source->clone_empty(); @@ -823,7 +826,8 @@ void configure_two_value_reader(BlockReader& reader, std::shared_ptr sour size_t batch_size = 16) { configure_reader(reader, source, batch_size, make_test_tablet_schema({{"val", FieldType::OLAP_FIELD_TYPE_BIGINT}, - {"val2", FieldType::OLAP_FIELD_TYPE_BIGINT}})); + {"val2", FieldType::OLAP_FIELD_TYPE_BIGINT}}), + {{1, 3}, {2, 4}}); } TEST_F(BlockReaderChangeNextBlockTest, MinDeltaNoOpUpdateComparesAllValueColumns) { @@ -993,7 +997,7 @@ TEST_F(BlockReaderChangeNextBlockTest, DetailUsesOrdinalBeforePairWhenNamesColli {{"v", FieldType::OLAP_FIELD_TYPE_BIGINT}, {changelog::build_before_column_name("v"), FieldType::OLAP_FIELD_TYPE_BIGINT}}); BlockReader reader; - configure_reader(reader, source, 16, std::move(schema)); + configure_reader(reader, source, 16, std::move(schema), {{1, 3}, {2, 4}}); bool eof = false; Block output = source->clone_empty(); diff --git a/be/test/storage/iterator/block_reader_changelog_vcollect_merge_test.cpp b/be/test/storage/iterator/block_reader_changelog_vcollect_merge_test.cpp index 3ee5d4f95482b0..12136961cc924c 100644 --- a/be/test/storage/iterator/block_reader_changelog_vcollect_merge_test.cpp +++ b/be/test/storage/iterator/block_reader_changelog_vcollect_merge_test.cpp @@ -176,7 +176,11 @@ class FakeRowsetReader : public RowsetReader { meta->set_num_rows(static_cast(_rows.size())); meta->set_rowset_type(BETA_ROWSET); _rowset = std::make_shared(nullptr, meta); - _read_schema = std::make_shared(make_changelog_schema()->columns()); + auto schema = make_changelog_schema(); + _read_schema = std::make_shared(schema->columns()); + EXPECT_TRUE( + _read_schema->init_changelog_column_mappings(*schema, {{VAL_IDX, BEFORE_VAL_IDX}}) + .ok()); } Status init(RowsetReaderContext*, const RowSetSplits&) override { return Status::OK(); } @@ -251,11 +255,12 @@ void configure_two_rowset_merge(BlockReader& reader, std::vector rowset0_ro reader._reader_type = ReaderType::READER_BASE_COMPACTION; reader._tablet_schema = make_changelog_schema(); - // Identity read schema over the full changelog layout. ReadSchema derives num_key_columns() - // from is_key() and tso_ordinal() from the CHANGELOG_TSO_COL name, which Level1Iterator::init() - // uses to pick the TSO as the merge sequence column, and which - // _validate_merge_compare_contract() checks for the leading key prefix. + // Scanner supplies pairs and resolves the system-column UIDs before merge initialization. reader._read_schema = std::make_shared(reader._tablet_schema->columns()); + ASSERT_TRUE(reader._read_schema + ->init_changelog_column_mappings(*reader._tablet_schema, + {{VAL_IDX, BEFORE_VAL_IDX}}) + .ok()); // Mirror BlockReader::_init_collect_iter: init the collect iterator (force_merge=true, // as a MIN_DELTA stream does), add one child per rowset, then build the heap. diff --git a/be/test/storage/mow/mow_transform_test_base.h b/be/test/storage/mow/mow_transform_test_base.h index a8bdbada85cbe4..60cc11e3c660d9 100644 --- a/be/test/storage/mow/mow_transform_test_base.h +++ b/be/test/storage/mow/mow_transform_test_base.h @@ -241,9 +241,9 @@ class MowTransformTestBase : public testing::Test { auto read_schema = std::make_shared(schema->columns()); RowsetReaderContext context; context.read_schema = read_schema; - static_cast(read_schema->init_from_tablet_schema(*schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false)); + static_cast( + read_schema->init_from_tablet_schema(*schema, + /*merge_by_sequence_mapping=*/false)); context.need_ordered_result = true; OlapReaderStatistics statistics; context.stats = &statistics; diff --git a/be/test/storage/read_schema_test.cpp b/be/test/storage/read_schema_test.cpp index d64965b20faf47..51834b4d8e19b7 100644 --- a/be/test/storage/read_schema_test.cpp +++ b/be/test/storage/read_schema_test.cpp @@ -80,9 +80,14 @@ TabletSchemaSPtr create_changelog_schema_with_colliding_names() { schema->append_column(*create_int_column(13, changelog::build_before_column_name("v"))); schema->append_column(*create_int_column( 14, changelog::build_before_column_name(changelog::build_before_column_name("v")))); - schema->append_column(*create_int_column(15, CHANGELOG_TSO_COL)); - schema->append_column(*create_int_column(16, CHANGELOG_LSN_COL)); - schema->append_column(*create_int_column(17, CHANGELOG_OP_COL)); + for (const auto& [uid, name] : + {std::pair {15, CHANGELOG_TSO_COL}, std::pair {16, CHANGELOG_LSN_COL}, + std::pair {17, CHANGELOG_OP_COL}}) { + auto column = create_int_column(uid, name); + column->set_type(FieldType::OLAP_FIELD_TYPE_BIGINT); + column->set_length(sizeof(int64_t)); + schema->append_column(*column); + } return schema; } @@ -157,48 +162,49 @@ TEST(ReadSchemaTest, AppendedDroppedColumnDoesNotExtendReadBlock) { EXPECT_EQ(2, read_schema.create_read_block().columns()); } -TEST(ReadSchemaTest, ChangelogMappingsUsePhysicalSchemaOrdinals) { - auto tablet_schema = create_changelog_schema_with_colliding_names(); - // Reorder the two AFTER values and their BEFORE companions to verify that ReadSchema stores - // dense read ordinals resolved by unique id, rather than physical tablet column ids or names. - ReadSchema read_schema(project_columns_by_ordinal( - tablet_schema->columns(), std::vector {0, 2, 1, 4, 3, 5, 6, 7})); - - EXPECT_TRUE(read_schema - .init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/true) - .ok()); - - EXPECT_TRUE(read_schema.changelog_value_pairs_complete()); - EXPECT_EQ(read_schema.changelog_value_column_pairs(), - (ReadSchema::ChangelogValueColumnPairs {{2, 4}, {1, 3}})); - EXPECT_EQ(4, read_schema.before_column_ordinal(2)); - EXPECT_EQ(3, read_schema.before_column_ordinal(1)); - for (ColumnId ordinal : {0, 3, 4, 5, 6, 7}) { - EXPECT_EQ(ordinal, read_schema.before_column_ordinal(ordinal)); - } +TEST(ReadSchemaTest, ExplicitChangelogMappingsIgnoreNamesAndOrder) { + ReadSchema schema({create_int_column(10, "key", true), + create_int_column(14, "arbitrary_before"), + create_int_column(11, "arbitrary_current"), create_int_column(12, "other")}); + ASSERT_TRUE(schema.init_changelog_column_mappings({{2, 1}}, -1, -1, -1).ok()); + EXPECT_EQ(1, schema.before_column_ordinal(2)); + EXPECT_EQ(3, schema.before_column_ordinal(3)); + schema.append_dropped_columns({*create_int_column(20, "private")}); + EXPECT_EQ(4, schema.num_block_columns()); + EXPECT_EQ(5, schema.num_read_columns()); + EXPECT_EQ(1, schema.before_column_ordinal(2)); + EXPECT_FALSE(schema.init_changelog_column_mappings({{2, 4}}, -1, -1, -1).ok()); } -TEST(ReadSchemaTest, MalformedChangelogLayoutKeepsConservativeNameMapping) { - TabletSchema tablet_schema; - tablet_schema.append_column(*create_int_column(10, "key", true)); - tablet_schema.append_column(*create_int_column(11, "v")); - tablet_schema.append_column(*create_int_column(12, changelog::build_before_column_name("v"))); - tablet_schema.append_column(*create_int_column(13, "orphan")); - tablet_schema.append_column(*create_int_column(14, CHANGELOG_TSO_COL)); - tablet_schema.append_column(*create_int_column(15, CHANGELOG_LSN_COL)); - tablet_schema.append_column(*create_int_column(16, CHANGELOG_OP_COL)); - ReadSchema read_schema(tablet_schema.columns()); - - EXPECT_TRUE(read_schema - .init_from_tablet_schema(tablet_schema, /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/true) - .ok()); +TEST(ReadSchemaTest, ChangelogMappingRejectsInvalidPairs) { + ReadSchema schema({create_int_column(10, "key", true), create_int_column(11, "value"), + create_int_column(12, "before"), create_struct_column(13)}); + EXPECT_FALSE(schema.init_changelog_column_mappings({{1, 1}}, -1, -1, -1).ok()); + EXPECT_FALSE(schema.init_changelog_column_mappings({{0, 2}}, -1, -1, -1).ok()); + EXPECT_FALSE(schema.init_changelog_column_mappings({{1, 2}, {2, 3}}, -1, -1, -1).ok()); + EXPECT_FALSE(schema.init_changelog_column_mappings({{1, 3}}, -1, -1, -1).ok()); + EXPECT_FALSE(schema.init_changelog_column_mappings({}, 4, -1, -1).ok()); + EXPECT_FALSE(schema.init_changelog_column_mappings({}, -2, -1, -1).ok()); + EXPECT_FALSE(schema.init_changelog_column_mappings({}, 1, 1, -1).ok()); + ASSERT_TRUE(schema.init_changelog_column_mappings({}, -1, -1, -1).ok()); + EXPECT_TRUE(schema.changelog_value_column_pairs().empty()); + EXPECT_EQ(1, schema.before_column_ordinal(1)); +} - EXPECT_FALSE(read_schema.changelog_value_pairs_complete()); - EXPECT_TRUE(read_schema.changelog_value_column_pairs().empty()); - EXPECT_EQ(2, read_schema.before_column_ordinal(1)); +TEST(ReadSchemaTest, ChangelogSystemOrdinalsResolveByUid) { + auto tablet = create_changelog_schema_with_colliding_names(); + auto tso = std::make_shared(*tablet->columns()[5]); + tso->set_name("renamed_tso"); + ReadSchema schema({tso, tablet->columns()[1], tablet->columns()[3]}); + ASSERT_TRUE(schema.init_changelog_column_mappings(*tablet, {{1, 2}}).ok()); + EXPECT_EQ(0, schema.tso_ordinal()); + EXPECT_EQ(-1, schema.op_ordinal()); + EXPECT_EQ(2, schema.before_column_ordinal(1)); + // Adding compaction row IDs must not reset an already initialized mapping. + schema.append_compaction_rowid_column(); + EXPECT_EQ(0, schema.tso_ordinal()); + EXPECT_EQ(2, schema.before_column_ordinal(1)); + EXPECT_EQ(3, schema.rowid_ordinal()); } } // namespace @@ -239,8 +245,7 @@ TEST(ReadSchemaTest, TabletHasSequenceMapIsIndependentOfTheProjection) { ReadSchema whole(tablet_schema->columns()); ASSERT_TRUE(whole.init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); EXPECT_TRUE(whole.tablet_has_sequence_map()); @@ -248,8 +253,7 @@ TEST(ReadSchemaTest, TabletHasSequenceMapIsIndependentOfTheProjection) { ReadSchema key_only( project_columns_by_ordinal(tablet_schema->columns(), std::vector {0})); ASSERT_TRUE(key_only.init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); EXPECT_TRUE(key_only.tablet_has_sequence_map()); } @@ -262,8 +266,7 @@ TEST(ReadSchemaTest, TabletWithoutSequenceMap) { ReadSchema read_schema(tablet_schema->columns()); ASSERT_TRUE(read_schema .init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); EXPECT_FALSE(read_schema.tablet_has_sequence_map()); EXPECT_FALSE(read_schema.tablet_has_extracted_variant_columns()); @@ -274,8 +277,7 @@ TEST(ReadSchemaTest, MergeBySequenceMappingBuildsTheMap) { ReadSchema read_schema(tablet_schema->columns()); ASSERT_TRUE(read_schema .init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/true, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/true) .ok()); // Ordinals are this ReadSchema's, not the tablet schema's column ids. @@ -289,8 +291,7 @@ TEST(ReadSchemaTest, MergeBySequenceMappingBuildsTheMap) { ReadSchema untouched(tablet_schema->columns()); ASSERT_TRUE(untouched .init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); EXPECT_TRUE(untouched.sequence_map().empty()); } @@ -303,8 +304,7 @@ TEST(ReadSchemaTest, SequenceColumnAndSequenceMapConflict) { ReadSchema read_schema(tablet_schema->columns()); auto st = read_schema.init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/true, - /*map_changelog_columns=*/false); + /*merge_by_sequence_mapping=*/true); EXPECT_FALSE(st.ok()) << st; EXPECT_TRUE(read_schema.sequence_map().empty()); } @@ -323,8 +323,7 @@ TEST(ReadSchemaTest, HasExtractedVariantColumns) { ReadSchema without_extracted(tablet_schema->columns()); ASSERT_TRUE(without_extracted .init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); EXPECT_FALSE(without_extracted.tablet_has_extracted_variant_columns()); @@ -341,8 +340,7 @@ TEST(ReadSchemaTest, HasExtractedVariantColumns) { project_columns_by_ordinal(tablet_schema->columns(), std::vector {0})); ASSERT_TRUE(read_schema .init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); EXPECT_TRUE(read_schema.tablet_has_extracted_variant_columns()); } diff --git a/be/test/storage/rowid_conversion_test.cpp b/be/test/storage/rowid_conversion_test.cpp index 76b25e2944d7d0..0f2cb8833d41c7 100644 --- a/be/test/storage/rowid_conversion_test.cpp +++ b/be/test/storage/rowid_conversion_test.cpp @@ -397,8 +397,7 @@ class TestRowIdConversion : public testing::TestWithParaminit_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); @@ -837,8 +836,7 @@ TEST_F(TestRowIdConversion, SingleRowsetGroupedCompactionRowIdConversionIsComple reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_reader; create_and_init_rowset_reader(output_rowset.get(), reader_context, &output_reader); @@ -1040,8 +1038,7 @@ TEST_F(TestRowIdConversion, SingleRowsetGroupedCompactionRowIdConversionIsComple second_reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr second_output_reader; create_and_init_rowset_reader(second_output_rowset.get(), second_reader_context, diff --git a/be/test/storage/rowset/segment_flusher_format_test.cpp b/be/test/storage/rowset/segment_flusher_format_test.cpp index 4c747eb816fc71..0284f9ee711cf8 100644 --- a/be/test/storage/rowset/segment_flusher_format_test.cpp +++ b/be/test/storage/rowset/segment_flusher_format_test.cpp @@ -3338,6 +3338,7 @@ class SegmentFlusherTransformFormatTest : public SegmentFlusherFormatTest { options.source.mow_context = std::move(source_mow_context); options.source.is_transient_rowset_writer = source_is_transient; options.source.source_write_type = DataWriteType::TYPE_DIRECT; + ASSERT_TRUE(testutil::resolve_test_changelog_layout(context).ok()); for (int64_t segment_id = 0; segment_id < 2; ++segment_id) { auto lsn_ids = std::make_shared>(); for (int64_t row = 0; row < rows_per_segment; ++row) { diff --git a/be/test/storage/storage_engine_test.cpp b/be/test/storage/storage_engine_test.cpp index 9099601d434620..0f914c9c6e372b 100644 --- a/be/test/storage/storage_engine_test.cpp +++ b/be/test/storage/storage_engine_test.cpp @@ -30,6 +30,7 @@ #include #include "common/status.h" +#include "cpp/sync_point.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" #include "storage/data_dir.h" @@ -141,8 +142,44 @@ TEST_F(StorageEngineTest, TestAsyncPublish) { TabletSharedPtr tablet = _storage_engine->tablet_manager()->get_tablet(tablet_id); EXPECT_EQ(tablet->max_version().second, 10); + auto mapping = std::make_shared(); + mapping->set_need_historical_value(true); + auto* entry = mapping->add_entries(); + entry->set_source_column_unique_id(1); + entry->set_current_column_unique_id(2); + { + SyncPoint::CallbackGuard guard; + SyncPoint::get_instance()->set_call_back( + "StorageEngine::add_async_publish_task::save_pending_publish_info", + [](auto&& args) { + auto* result = try_any_cast_ret(args); + result->first = + Status::Error("injected pending-publish write failure"); + result->second = true; + }, + &guard); + SyncPoint::get_instance()->enable_processing(); + auto failed = _storage_engine->add_async_publish_task(partition_id, tablet_id, 11, 11, + false, 110, mapping); + EXPECT_TRUE(failed.is()) << failed; + EXPECT_TRUE(_storage_engine->_async_publish_tasks.empty()); + SyncPoint::get_instance()->disable_processing(); + } + bool persisted_after_failure = false; + ASSERT_TRUE( + TabletMetaManager::traverse_pending_publish(_data_dir->get_meta(), + [&](int64_t, int64_t, std::string_view) { + persisted_after_failure = true; + return true; + }) + .ok()); + EXPECT_FALSE(persisted_after_failure); + for (int64_t i = 5; i < 12; ++i) { - _storage_engine->add_async_publish_task(partition_id, tablet_id, i, i, false, i * 10); + EXPECT_TRUE(_storage_engine + ->add_async_publish_task(partition_id, tablet_id, i, i, false, i * 10, + mapping) + .ok()); } EXPECT_EQ(_storage_engine->_async_publish_tasks[tablet_id].size(), 7); EXPECT_EQ(_storage_engine->get_pending_publish_min_version(tablet_id), 5); @@ -158,11 +195,25 @@ TEST_F(StorageEngineTest, TestAsyncPublish) { bool parsed = pb.ParseFromArray(info.data(), static_cast(info.size())); EXPECT_TRUE(parsed); version_to_commit_tso[publish_version] = pb.commit_tso(); + EXPECT_TRUE(pb.has_changelog_column_mappings()); + EXPECT_EQ(pb.changelog_column_mappings().SerializeAsString(), + mapping->SerializeAsString()); + // DataDir uses this same recovery entry point after deserializing PB. + EXPECT_TRUE(_storage_engine + ->add_async_publish_task( + pb.partition_id(), tablet_id, publish_version, + pb.transaction_id(), true, pb.commit_tso(), + std::make_shared( + pb.changelog_column_mappings())) + .ok()); return true; }); EXPECT_TRUE(st.ok()) << st; EXPECT_EQ(version_to_commit_tso[5], 50); EXPECT_EQ(version_to_commit_tso[11], 110); + EXPECT_EQ( + std::get<3>(_storage_engine->_async_publish_tasks[tablet_id][11])->SerializeAsString(), + mapping->SerializeAsString()); for (int64_t i = 1; i < 8; ++i) { _storage_engine->_process_async_publish(); @@ -172,13 +223,19 @@ TEST_F(StorageEngineTest, TestAsyncPublish) { EXPECT_EQ(_storage_engine->_async_publish_tasks.size(), 0); for (int64_t i = 100; i < config::max_tablet_version_num + 120; ++i) { - _storage_engine->add_async_publish_task(partition_id, tablet_id, i, i, false, -1 /*tso*/); + EXPECT_TRUE( + _storage_engine + ->add_async_publish_task(partition_id, tablet_id, i, i, false, -1 /*tso*/) + .ok()); } EXPECT_EQ(_storage_engine->_async_publish_tasks[tablet_id].size(), config::max_tablet_version_num + 20); for (int64_t i = 90; i < 120; ++i) { - _storage_engine->add_async_publish_task(partition_id, tablet_id, i, i, false, -1 /*tso*/); + EXPECT_TRUE( + _storage_engine + ->add_async_publish_task(partition_id, tablet_id, i, i, false, -1 /*tso*/) + .ok()); } EXPECT_EQ(_storage_engine->_async_publish_tasks[tablet_id].size(), config::max_tablet_version_num + 30); @@ -189,11 +246,21 @@ TEST_F(StorageEngineTest, TestAsyncPublish) { config::max_tablet_version_num); EXPECT_EQ(_storage_engine->get_pending_publish_min_version(tablet_id), 120); + _storage_engine->_tablet_publish_txn_thread_pool->shutdown(); + EXPECT_TRUE( + _storage_engine + ->add_async_publish_task(partition_id, tablet_id, 11, 11, false, 110, mapping) + .ok()); + _storage_engine->_process_async_publish(); + EXPECT_EQ(_storage_engine->_async_publish_tasks[tablet_id].size(), + config::max_tablet_version_num + 1); + EXPECT_EQ(std::get<3>(_storage_engine->_async_publish_tasks[tablet_id][11]), mapping); + st = _storage_engine->tablet_manager()->drop_tablet(tablet_id, 0, false); EXPECT_EQ(st, Status::OK()); EXPECT_EQ(_storage_engine->_async_publish_tasks[tablet_id].size(), - config::max_tablet_version_num); + config::max_tablet_version_num + 1); _storage_engine->_process_async_publish(); EXPECT_EQ(_storage_engine->_async_publish_tasks.size(), 0); } diff --git a/be/test/storage/transform/changelog_derive_test.cpp b/be/test/storage/transform/changelog_derive_test.cpp index 73c29e7cbee37a..5b5584bd8e288c 100644 --- a/be/test/storage/transform/changelog_derive_test.cpp +++ b/be/test/storage/transform/changelog_derive_test.cpp @@ -334,6 +334,7 @@ TEST_F(ChangelogDeriveTest, PlainAppendAndDeleteWithAfterAndLsn) { cfg.write_before = false; // no PU, no BEFORE -> plain derive register_segment_lsns(rwc, 2); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"PlainChangelogDerive"})); @@ -402,6 +403,7 @@ TEST_F(ChangelogDeriveTest, PlainNoDeleteSignColumnAllAppend) { cfg.write_before = false; register_segment_lsns(rwc, 2); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"PlainChangelogDerive"})); @@ -503,6 +505,7 @@ TEST_F(ChangelogDeriveTest, PlainMapsHiddenKeysAndSkipsHiddenNonKeys) { } } + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(changelog_schema, &rwc); ASSERT_TRUE(chain.apply(ctx, &block).ok()); @@ -580,6 +583,7 @@ TEST_F(ChangelogDeriveTest, MowUpdateAndAppendTakesHistoryV2) { cfg.write_before = false; register_segment_lsns(rwc, 2); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"MowChangelogDerive"})); @@ -720,6 +724,7 @@ TEST_F(ChangelogDeriveTest, MowHiddenKeyColumnIsPartOfTheProbeKey) { cfg.write_before = false; register_segment_lsns(rwc, 1); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(changelog_schema, &rwc); ctx.tablet = probe_tablet; @@ -799,6 +804,7 @@ TEST_F(ChangelogDeriveTest, MowPartialUpdateWithBeforeImage) { cfg.write_before = true; register_segment_lsns(rwc, 2); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"MowChangelogDerive"})); @@ -895,6 +901,7 @@ TEST_F(ChangelogDeriveTest, MowInsertAfterDeletedHistoryRowIsAppend) { cfg.write_before = false; register_segment_lsns(rwc, 2); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(changelog_schema, &rwc); ctx.tablet = probe_tablet; @@ -982,6 +989,7 @@ TEST_F(ChangelogDeriveTest, MowLooksUpHistoryInBaseTabletNotChangelogTablet) { cfg.write_before = false; register_segment_lsns(rwc, 1); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); TransformExecContext ctx = exec_ctx(changelog_schema, &rwc); ctx.tablet = changelog_tablet; // NOT the tablet holding the history @@ -1058,6 +1066,7 @@ TEST_F(ChangelogDeriveTest, MowDeleteExistingAndNewKey) { cfg.write_before = false; register_segment_lsns(rwc, 2); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"MowChangelogDerive"})); @@ -1153,6 +1162,7 @@ TEST_F(ChangelogDeriveTest, MowBeforeImageMirrorsHistory) { .ok()); cfg.source.partial_update_info = pui; + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); // write_before -> MoW derive EXPECT_EQ(chain.stage_names(), (std::vector {"MowChangelogDerive"})); @@ -1265,6 +1275,7 @@ TEST_F(ChangelogDeriveTest, MowBeforeNoopZeroValueColumns) { cfg.write_before = true; register_segment_lsns(rwc, 2); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"MowChangelogDerive"})); @@ -1354,6 +1365,7 @@ TEST_F(ChangelogDeriveTest, MowSeqSourceSeqLosesStillReadsHistory) { cfg.write_before = false; register_segment_lsns(rwc, 2); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"MowChangelogDerive"})); @@ -1438,6 +1450,7 @@ TEST_F(ChangelogDeriveTest, MowMaterializesFlexiblePartialUpdate) { cfg.write_before = false; register_segment_lsns(rwc, 1); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"MowChangelogDerive"})); @@ -1502,6 +1515,7 @@ TEST_F(ChangelogDeriveTest, MowFixedPartialUpdateRejectsBadWidth) { cfg.source.mow_context = mow; cfg.write_before = false; + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"MowChangelogDerive"})); @@ -1595,6 +1609,7 @@ TEST_F(ChangelogDeriveTest, RejectsNegativeSegmentId) { cfg.write_before = false; register_segment_lsns(rwc, 1); + ASSERT_TRUE(testutil::resolve_test_changelog_layout(rwc).ok()); auto chain = build_transform_chain(rwc); EXPECT_EQ(chain.stage_names(), (std::vector {"PlainChangelogDerive"})); @@ -1634,18 +1649,7 @@ TEST_F(ChangelogDeriveTest, RejectsSchemaAndWriteBeforeDisagreement) { cfg.write_before = true; // the schema has no BEFORE columns register_segment_lsns(rwc, 1); - auto chain = build_transform_chain(rwc); - TransformExecContext ctx = exec_ctx(changelog_schema, &rwc); - ctx.tablet = tablets.changelog; - ctx.mow_context = cfg.source.mow_context; - - Block block = source_schema->create_storage_block(); - for (size_t cid = 0; cid < source_schema->num_columns(); ++cid) { - block.get_by_position(cid).column->assert_mutable()->insert_default(); - } - auto st = chain.apply(ctx, &block); - EXPECT_FALSE(st.ok()); - EXPECT_NE(st.to_string().find("does not match the derived block"), std::string::npos) << st; + EXPECT_FALSE(testutil::resolve_test_changelog_layout(rwc).ok()); } // The TSO/LSN/OP positions come from the column names. A changelog schema whose @@ -1680,19 +1684,110 @@ TEST_F(ChangelogDeriveTest, RejectsSchemaWithoutHiddenColumns) { cfg.write_before = false; register_segment_lsns(rwc, 1); - auto chain = build_transform_chain(rwc); - EXPECT_EQ(chain.stage_names(), (std::vector {"PlainChangelogDerive"})); - - TransformExecContext ctx = exec_ctx(changelog_schema, &rwc); - ctx.tablet = tablets.changelog; - - Block block = source_schema->create_storage_block(); - for (size_t cid = 0; cid < source_schema->num_columns(); ++cid) { - block.get_by_position(cid).column->assert_mutable()->insert_default(); - } - auto st = chain.apply(ctx, &block); + auto st = testutil::resolve_test_changelog_layout(rwc); EXPECT_FALSE(st.ok()); EXPECT_NE(st.to_string().find("misses its hidden columns"), std::string::npos) << st; } +// The wire mapping is independent of both names and target physical order. +TEST_F(ChangelogDeriveTest, ResolveExplicitInterleavedMapping) { + auto source = create_changelog_pu_source_schema(); + auto target = create_changelog_before_schema(); + PChangelogWriteColumnMappings mapping; + mapping.set_need_historical_value(true); + for (int cid = 0; cid < 3; ++cid) { + auto* entry = mapping.add_entries(); + entry->set_source_column_unique_id(source->column(cid).unique_id()); + entry->set_current_column_unique_id(target->column(cid).unique_id()); + if (cid > 0) { + entry->set_before_column_unique_id(target->column(cid + 2).unique_id()); + } + } + TabletSchemaPB target_pb; + target->to_schema_pb(&target_pb); + target_pb.mutable_column()->SwapElements(1, 4); + auto reordered = std::make_shared(); + reordered->init_from_pb(target_pb); + std::shared_ptr layout; + ASSERT_TRUE(changelog::resolve_layout(source, reordered, mapping, &layout).ok()); + EXPECT_EQ(layout->current_target_cids, (std::vector {0, 4, 2})); + EXPECT_EQ(layout->before_target_cids, (std::vector {3, 1})); + EXPECT_TRUE(layout->need_historical_value); + auto invalid = mapping; + invalid.mutable_entries(1)->set_current_column_unique_id(99999); + EXPECT_FALSE(changelog::resolve_layout(source, reordered, invalid, &layout).ok()); + invalid = mapping; + invalid.mutable_entries(1)->set_current_column_unique_id( + invalid.entries(0).current_column_unique_id()); + EXPECT_FALSE(changelog::resolve_layout(source, reordered, invalid, &layout).ok()); + invalid = mapping; + invalid.mutable_entries(0)->clear_source_column_unique_id(); + EXPECT_FALSE(changelog::resolve_layout(source, reordered, invalid, &layout).ok()); + invalid = mapping; + invalid.set_need_historical_value(false); + EXPECT_FALSE(changelog::resolve_layout(source, reordered, invalid, &layout).ok()); + invalid = mapping; + invalid.clear_need_historical_value(); + EXPECT_FALSE(changelog::resolve_layout(source, reordered, invalid, &layout).ok()); +} + +TEST_F(ChangelogDeriveTest, ResolveRejectsShapeAndNullableNarrowing) { + auto source = create_changelog_pu_source_schema(); + auto target = create_changelog_before_schema(); + auto mapping = testutil::create_changelog_mapping(*source, *target, true); + TabletSchemaPB pb; + target->to_schema_pb(&pb); + pb.mutable_column(1)->set_is_nullable(false); + auto invalid_target = std::make_shared(); + invalid_target->init_from_pb(pb); + std::shared_ptr layout; + EXPECT_FALSE(changelog::resolve_layout(source, invalid_target, mapping, &layout).ok()); + target->to_schema_pb(&pb); + pb.mutable_column(1)->set_type("BIGINT"); + invalid_target->init_from_pb(pb); + EXPECT_FALSE(changelog::resolve_layout(source, invalid_target, mapping, &layout).ok()); + source->to_schema_pb(&pb); + pb.mutable_column(1)->set_is_nullable(false); + auto nonnullable_source = std::make_shared(); + nonnullable_source->init_from_pb(pb); + ASSERT_TRUE(changelog::resolve_layout(nonnullable_source, target, mapping, &layout).ok()); + EXPECT_TRUE(layout->current_nullable_widening[1]); + EXPECT_FALSE(layout->current_nullable_widening[0]); + target->to_schema_pb(&pb); + pb.mutable_column(0)->set_is_nullable(true); + auto nullable_key_target = std::make_shared(); + nullable_key_target->init_from_pb(pb); + EXPECT_FALSE(changelog::resolve_layout(source, nullable_key_target, mapping, &layout).ok()); +} + +TEST_F(ChangelogDeriveTest, PlainInterleavedTargetPreservesValues) { + auto source = create_changelog_pu_source_schema(); + auto tablets = create_changelog_tablets(8991, TKeysType::DUP_KEYS, false); + TabletSchemaPB pb; + tablets.changelog->tablet_schema()->to_schema_pb(&pb); + pb.mutable_column()->SwapElements(1, 4); // v1 crosses a system column + auto target = std::make_shared(); + target->init_from_pb(pb); + RowsetWriterContext context; + context.tablet_schema = target; + context.write_changelog_opt().enable = true; + auto& cfg = context.write_changelog_opt().write_changelog_config(); + cfg.source.tablet_schema = source; + ASSERT_TRUE(testutil::resolve_test_changelog_layout(context).ok()); + register_segment_lsns(context, 1); + Block block = source->create_storage_block(); + block.get_by_position(0).column->assert_mutable()->insert(Field(int64_t(7))); + block.get_by_position(1).column->assert_mutable()->insert(Field(int64_t(11))); + block.get_by_position(2).column->assert_mutable()->insert(Field(int64_t(13))); + block.get_by_position(3).column->assert_mutable()->insert_default(); + auto ctx = exec_ctx(target, &context); + auto chain = build_transform_chain(context); + ASSERT_TRUE(chain.apply(ctx, &block).ok()); + EXPECT_EQ(block.get_by_position(4).column->get_int(0), 11); + EXPECT_EQ(block.get_by_position(2).column->get_int(0), 13); + EXPECT_EQ(block.get_by_position(0).column->get_int(0), 7); + EXPECT_EQ(read_op(block, target->changelog_op_col_idx(), 0), + static_cast(ChangelogOp::APPEND)); +} + } // namespace doris diff --git a/be/test/storage/transform/validate_stage_test.cpp b/be/test/storage/transform/validate_stage_test.cpp index 7691ebcfbe2bc5..bb6daf89d8a95b 100644 --- a/be/test/storage/transform/validate_stage_test.cpp +++ b/be/test/storage/transform/validate_stage_test.cpp @@ -188,11 +188,13 @@ TEST_F(ValidateStageTest, CompositionChangelogTransientPartialUpdateStaysPlain) EXPECT_EQ(build_transform_chain(rwc).stage_names(), (V {"PlainChangelogDerive"})); // the same write with a BEFORE image still needs the probe - rwc.write_changelog_opt().write_changelog_config().write_before = true; + auto layout = std::make_shared(); + layout->need_historical_value = true; + rwc.write_changelog_opt().write_changelog_config().layout = layout; EXPECT_EQ(build_transform_chain(rwc).stage_names(), (V {"MowChangelogDerive"})); // and a non-transient write of the same partial update does probe - rwc.write_changelog_opt().write_changelog_config().write_before = false; + rwc.write_changelog_opt().write_changelog_config().layout.reset(); rwc.write_changelog_opt().write_changelog_config().source.is_transient_rowset_writer = false; EXPECT_EQ(build_transform_chain(rwc).stage_names(), (V {"MowChangelogDerive"})); } @@ -204,7 +206,9 @@ TEST_F(ValidateStageTest, CompositionChangelogBeforeImagePicksMowDerive) { auto schema = create_mow_schema(/*has_seq=*/false); RowsetWriterContext rwc = direct_rwc(schema); rwc.write_changelog_opt().enable = true; - rwc.write_changelog_opt().write_changelog_config().write_before = true; + auto layout = std::make_shared(); + layout->need_historical_value = true; + rwc.write_changelog_opt().write_changelog_config().layout = layout; EXPECT_EQ(build_transform_chain(rwc).stage_names(), (V {"MowChangelogDerive"})); } diff --git a/be/test/storage/txn/txn_manager_test.cpp b/be/test/storage/txn/txn_manager_test.cpp index a133f6489c2da5..b2bb96e361af42 100644 --- a/be/test/storage/txn/txn_manager_test.cpp +++ b/be/test/storage/txn/txn_manager_test.cpp @@ -44,6 +44,7 @@ #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_schema.h" #include "storage/task/engine_publish_version_task.h" +#include "testutil/creators.h" #include "util/uid_util.h" using ::testing::_; @@ -124,7 +125,8 @@ class TxnManagerTest : public testing::Test { return json_rowset_meta.substr(0, json_rowset_meta.size() - 1); } - RowsetSharedPtr create_changelog_rowset(int64_t rowset_id, const Version& version) { + RowsetSharedPtr create_changelog_rowset(int64_t rowset_id, const Version& version, + bool with_system_columns = true) { auto json_rowset_meta = read_rowset_meta_json(rowset_meta_path); RowsetMetaSharedPtr rowset_meta(new RowsetMeta()); rowset_meta->init_from_json(json_rowset_meta); @@ -138,9 +140,28 @@ class TxnManagerTest : public testing::Test { rowset_meta->set_version(version); rowset_meta->mark_changelog(); + TabletSchemaPB schema_pb; + _schema->to_schema_pb(&schema_pb); + schema_pb.set_keys_type(DUP_KEYS); + if (with_system_columns) { + int uid = 100; + for (const auto& name : {CHANGELOG_TSO_COL, CHANGELOG_LSN_COL, CHANGELOG_OP_COL}) { + auto* column = schema_pb.add_column(); + column->set_unique_id(uid++); + column->set_name(name); + column->set_type("BIGINT"); + column->set_is_key(false); + column->set_is_nullable(name == CHANGELOG_TSO_COL); + column->set_length(8); + column->set_index_length(8); + } + } + auto changelog_schema = std::make_shared(); + changelog_schema->init_from_pb(schema_pb); + rowset_meta->set_tablet_schema(changelog_schema); RowsetSharedPtr rowset; - EXPECT_EQ(Status::OK(), - RowsetFactory::create_rowset(_schema, rowset_meta_path, rowset_meta, &rowset)); + EXPECT_EQ(Status::OK(), RowsetFactory::create_rowset(changelog_schema, rowset_meta_path, + rowset_meta, &rowset)); return rowset; } @@ -481,11 +502,19 @@ TEST_F(TxnManagerTest, TxnWithChangelog) { EXPECT_EQ(committed_changelog_meta->rowset_state(), RowsetStatePB::COMMITTED); EXPECT_TRUE(committed_changelog_meta->is_changelog()); + auto mappings = std::make_shared( + testutil::create_changelog_mapping(*_schema, *changelog_rowset->tablet_schema(), + false)); Version new_version(10, 10); TabletPublishStatistics stats; std::shared_ptr tablet_txn_info = nullptr; st = k_engine->txn_manager()->publish_txn(_meta.get(), partition_id, transaction_id, tablet_id, _tablet_uid, new_version, &stats, tablet_txn_info); + ASSERT_FALSE(st.ok()); + EXPECT_EQ(changelog_rowset->rowset_meta_state(), RowsetStatePB::COMMITTED); + st = k_engine->txn_manager()->publish_txn(_meta.get(), partition_id, transaction_id, tablet_id, + _tablet_uid, new_version, &stats, tablet_txn_info, + 12345, mappings); ASSERT_TRUE(st.ok()) << st; RowsetMetaSharedPtr rowset_meta(new RowsetMeta()); @@ -509,7 +538,7 @@ TEST_F(TxnManagerTest, TxnWithChangelog) { // rewrite, which reads the LSN of conflicting rows from that rowset, must fail with an error // instead of indexing past the schema. TEST_F(TxnManagerTest, PublishRejectsChangelogRowsetWithoutHiddenColumns) { - auto changelog_rowset = create_changelog_rowset(30001, _rowset->version()); + auto changelog_rowset = create_changelog_rowset(30001, _rowset->version(), false); ASSERT_LT(changelog_rowset->tablet_schema()->changelog_lsn_col_idx(), 0); auto tablet = k_engine->tablet_manager()->get_tablet(tablet_id); ASSERT_TRUE(tablet != nullptr); @@ -521,7 +550,11 @@ TEST_F(TxnManagerTest, PublishRejectsChangelogRowsetWithoutHiddenColumns) { TabletTxnInfo txn_info(load_id, _rowset); txn_info.attach_changelog.rowset = changelog_rowset; txn_info.attach_changelog.tablet = tablet; - auto st = BaseTablet::update_delete_bitmap(tablet, &txn_info, transaction_id); + auto mapping = + testutil::create_changelog_mapping(*_schema, *changelog_rowset->tablet_schema(), false); + std::shared_ptr layout; + auto st = + changelog::resolve_layout(_schema, changelog_rowset->tablet_schema(), mapping, &layout); ASSERT_FALSE(st.ok()); EXPECT_NE(st.to_string().find("misses its hidden columns"), std::string::npos) << st; } @@ -684,4 +717,24 @@ TEST_F(TxnManagerTest, DeleteCommittedTxnCleanupOnError) { ASSERT_TRUE(found_txn_ids.empty()) << "Transaction should not be found in tablet related txns"; } +TEST_F(TxnManagerTest, PendingPublishRetainsExplicitMapping) { + auto rowset = create_changelog_rowset(30003, _rowset->version()); + PendingPublishInfoPB pending; + pending.set_partition_id(partition_id); + pending.set_transaction_id(transaction_id); + pending.set_commit_tso(12345); + *pending.mutable_changelog_column_mappings() = + testutil::create_changelog_mapping(*_schema, *rowset->tablet_schema(), true); + PendingPublishInfoPB recovered; + ASSERT_TRUE(recovered.ParseFromString(pending.SerializeAsString())); + ASSERT_TRUE(recovered.has_changelog_column_mappings()); + const auto& mapping = recovered.changelog_column_mappings(); + EXPECT_TRUE(mapping.has_need_historical_value()); + EXPECT_TRUE(mapping.need_historical_value()); + std::shared_ptr layout; + ASSERT_TRUE(changelog::resolve_layout(_schema, rowset->tablet_schema(), mapping, &layout).ok()); + EXPECT_TRUE(layout->need_historical_value); + EXPECT_TRUE(layout->before_source_cids.empty()); // key-only historical remains explicit +} + } // namespace doris diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index 0e8ebdaf46eb60..4438f43b1acd97 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -1494,8 +1494,7 @@ class VariantColumnWriterReaderTest : public testing::Test { reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*_tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RETURN_IF_ERROR(reader->init(&reader_context)); diff --git a/be/test/storage/variant/variant_doc_mode_compaction_test.cpp b/be/test/storage/variant/variant_doc_mode_compaction_test.cpp index 351c1db4c6a41e..fe698009be3003 100644 --- a/be/test/storage/variant/variant_doc_mode_compaction_test.cpp +++ b/be/test/storage/variant/variant_doc_mode_compaction_test.cpp @@ -430,8 +430,7 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen input_reader_context.read_schema = input_read_schema; EXPECT_TRUE(input_read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr input_rs_reader; create_and_init_rowset_reader(rowset.get(), input_reader_context, &input_rs_reader); @@ -483,8 +482,7 @@ TEST_F(VariantDocModeCompactionTest, variant_doc_mode_compaction_merge_10_segmen reader_context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); RowsetReaderSharedPtr output_rs_reader; create_and_init_rowset_reader(out_rowset.get(), reader_context, &output_rs_reader); diff --git a/be/test/testutil/creators.h b/be/test/testutil/creators.h index 9351d353fe1f92..2d727495471cf0 100644 --- a/be/test/testutil/creators.h +++ b/be/test/testutil/creators.h @@ -37,6 +37,7 @@ #include "runtime/descriptors.h" #include "runtime/query_context.h" #include "storage/changelog.h" +#include "storage/rowset/rowset_writer_context.h" #include "storage/tablet_info.h" #include "util/uid_util.h" @@ -189,11 +190,48 @@ inline TCreateTabletReq create_tablet_request( request.__set_storage_format(storage_format); request.tablet_schema.columns.reserve(column_defs.size()); for (const auto& column_def : column_defs) { - request.tablet_schema.columns.push_back(create_tablet_column(column_def)); + auto column = create_tablet_column(column_def); + column.__set_col_unique_id(request.tablet_schema.columns.size()); + request.tablet_schema.columns.push_back(std::move(column)); } return request; } +// Test planning boundary: produce explicit UID relationships from fixture names. +inline PChangelogWriteColumnMappings create_changelog_mapping(const TabletSchema& source, + const TabletSchema& target, + bool historical) { + PChangelogWriteColumnMappings mapping; + mapping.set_need_historical_value(historical); + for (uint32_t cid = 0; cid < source.num_columns(); ++cid) { + const auto& col = source.column(cid); + if (!col.visible() && !col.is_key()) { + continue; + } + auto* entry = mapping.add_entries(); + entry->set_source_column_unique_id(col.unique_id()); + const int current_cid = target.field_index(col.name()); + EXPECT_GE(current_cid, 0); + entry->set_current_column_unique_id( + current_cid < 0 ? -1000 : target.column(current_cid).unique_id()); + if (historical && !col.is_key()) { + const int before_cid = + target.field_index(changelog::build_before_column_name(col.name())); + entry->set_before_column_unique_id( + before_cid < 0 ? -1000 : target.column(before_cid).unique_id()); + } + } + return mapping; +} + +inline Status resolve_test_changelog_layout(RowsetWriterContext& context) { + auto& cfg = context.write_changelog_opt().write_changelog_config(); + const auto mapping = create_changelog_mapping(*cfg.source.tablet_schema, *context.tablet_schema, + cfg.write_before); + return changelog::resolve_layout(cfg.source.tablet_schema, context.tablet_schema, mapping, + &cfg.layout); +} + inline void enable_changelog(TCreateTabletReq* request) { DCHECK(request != nullptr); @@ -228,6 +266,9 @@ inline TTabletSchema create_changelog_tablet_schema(const TTabletSchema& base_sc {CHANGELOG_LSN_COL, TPrimitiveType::BIGINT, false, true, TAggregationType::NONE})); changelog_schema.columns.push_back(create_tablet_column( {CHANGELOG_OP_COL, TPrimitiveType::BIGINT, false, true, TAggregationType::NONE})); + for (size_t cid = 0; cid < changelog_schema.columns.size(); ++cid) { + changelog_schema.columns[cid].__set_col_unique_id(cid); + } return changelog_schema; } @@ -269,6 +310,28 @@ inline std::shared_ptr create_table_schema_param( } if (changelog_index_id > 0) { tschema.indexes[0].__set_changelog_index_id(changelog_index_id); + TChangelogWriteColumnMappings mapping; + mapping.__set_need_historical_value(false); + mapping.__set_entries({}); + for (const auto& source : columns) { + if (!source.visible && !source.is_key) { + continue; + } + TChangelogWriteColumnMapping entry; + entry.__set_source_column_unique_id(source.col_unique_id); + for (const auto& current : *changelog_columns) { + if (current.column_name == source.column_name) { + entry.__set_current_column_unique_id(current.col_unique_id); + } + if (current.column_name == + changelog::build_before_column_name(source.column_name)) { + entry.__set_before_column_unique_id(current.col_unique_id); + mapping.__set_need_historical_value(true); + } + } + mapping.entries.push_back(entry); + } + tschema.indexes[0].__set_changelog_column_mappings(mapping); TOlapTableIndexSchema changelog_index_schema; changelog_index_schema.id = changelog_index_id; changelog_index_schema.schema_hash = changelog_schema_hash; diff --git a/be/test/testutil/index_storage_test_util.cpp b/be/test/testutil/index_storage_test_util.cpp index 9f840d5de9d4ba..da7fbaeb2edfaf 100644 --- a/be/test/testutil/index_storage_test_util.cpp +++ b/be/test/testutil/index_storage_test_util.cpp @@ -1309,8 +1309,7 @@ Result IndexStorageTestFixture::read_rowsets( context.read_schema = read_schema; EXPECT_TRUE(read_schema ->init_from_tablet_schema(*_tablet_schema, - /*merge_by_sequence_mapping=*/false, - /*map_changelog_columns=*/false) + /*merge_by_sequence_mapping=*/false) .ok()); context.predicates = &predicates; context.stats = &result.stats; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java index 8cef574c204db2..1f1f667d2fe528 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java @@ -25,6 +25,7 @@ import org.apache.doris.cooldown.CooldownConf; import org.apache.doris.master.PartitionInfoCollector.PartitionCollectInfo; import org.apache.doris.task.PublishVersionTask; +import org.apache.doris.task.PublishVersionTask.RepublishInfo; import org.apache.doris.thrift.TPartitionVersionInfo; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TTablet; @@ -103,7 +104,7 @@ public void tabletReport(long backendId, Map backendTablets, Set tabletFoundInMeta, ListMultimap tabletMigrationMap, Map partitionVersionSyncMap, - Map> transactionsToPublish, + Map> transactionsToPublish, SetMultimap transactionsToClear, ListMultimap tabletRecoveryMap, List tabletToUpdate, @@ -153,7 +154,7 @@ private void processTabletReportAsync(long backendId, Map backend Set tabletFoundInMeta, ListMultimap tabletMigrationMap, Map partitionVersionSyncMap, - Map> transactionsToPublish, + Map> transactionsToPublish, SetMultimap transactionsToClear, ListMultimap tabletRecoveryMap, List tabletToUpdate, @@ -215,7 +216,7 @@ private void processTabletEntry(long backendId, Map backendTablet ListMultimap tabletDeleteFromMeta, Set tabletFoundInMeta, ListMultimap tabletMigrationMap, - Map> transactionsToPublish, + Map> transactionsToPublish, SetMultimap transactionsToClear, ListMultimap tabletRecoveryMap, List tabletToUpdate, @@ -252,7 +253,7 @@ private void processExistingTablet(long backendId, long tabletId, Replica replic HashMap storageMediumMap, ListMultimap tabletSyncMap, ListMultimap tabletMigrationMap, - Map> transactionsToPublish, + Map> transactionsToPublish, SetMultimap transactionsToClear, ListMultimap tabletRecoveryMap, List tabletToUpdate, @@ -465,7 +466,7 @@ private void logTabletReportSummary(long backendId, long feTabletNum, Set tabletFoundInMeta, ListMultimap tabletMigrationMap, Map partitionVersionSyncMap, - Map> transactionsToPublish, + Map> transactionsToPublish, SetMultimap transactionsToClear, List tabletToUpdate, ListMultimap tabletRecoveryMap, @@ -493,7 +494,7 @@ private void logTabletReportSummary(long backendId, long feTabletNum, } private void handleBackendTransactions(long backendId, List transactionIds, long tabletId, - TabletMeta tabletMeta, Map> transactionsToPublish, + TabletMeta tabletMeta, Map> transactionsToPublish, SetMultimap transactionsToClear) { GlobalTransactionMgrIface transactionMgr = Env.getCurrentGlobalTransactionMgr(); long partitionId = tabletMeta.getPartitionId(); @@ -557,18 +558,19 @@ private TPartitionVersionInfo generatePartitionVersionInfoWhenReport(Transaction } private void publishPartition(TransactionState transactionState, long transactionId, TabletMeta tabletMeta, - long partitionId, Map> transactionsToPublish) { + long partitionId, Map> transactionsToPublish) { TPartitionVersionInfo versionInfo = generatePartitionVersionInfoWhenReport(transactionState, transactionId, tabletMeta, partitionId); if (versionInfo != null) { synchronized (transactionsToPublish) { - SetMultimap map = transactionsToPublish.get( + SetMultimap map = transactionsToPublish.get( transactionState.getDbId()); if (map == null) { map = LinkedHashMultimap.create(); transactionsToPublish.put(transactionState.getDbId(), map); } - map.put(transactionId, versionInfo); + map.put(transactionId, new RepublishInfo(versionInfo, + transactionState.getChangelogWriteMappings(transactionId))); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapChangelogTableWrapper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapChangelogTableWrapper.java index a10601a8775813..68513416ff0be0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapChangelogTableWrapper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapChangelogTableWrapper.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -57,6 +58,28 @@ public OlapChangelogTableWrapper(OlapTable originTable, Map> getChangelogReadColumnPairs() { + if (!originTable.getChangelogConfig().getNeedHistoricalValue()) { + return Collections.emptyList(); + } + List> pairs = new ArrayList<>(); + for (Column source : originTable.getBaseSchema()) { + if (source.isKey() || !source.isVisible()) { + continue; + } + Column current = getColumn(source.getName()); + Column before = getColumn(Column.generateBeforeColName(source.getName())); + Preconditions.checkState(current != null && before != null, + "Missing changelog read pair for column %s", source.getName()); + pairs.add(Pair.of(current, before)); + } + return Collections.unmodifiableList(pairs); + } + @Override public long getBaseIndexId() { return changelogIndexMeta.getIndexId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 09420f3fabfdba..d15839fc02c6a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3417,9 +3417,9 @@ public void setReplicaAllocation(Map properties) { tableProperty.buildReplicaAllocation(); } - // for light schema change + // Changelog writes also require stable source UIDs when light schema change DDL is disabled. public void initSchemaColumnUniqueId() { - if (!getEnableLightSchemaChange()) { + if (!getEnableLightSchemaChange() && !hasChangelog()) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java index ab4b4030ee847a..111d664f821353 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java @@ -21,7 +21,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.cooldown.CooldownConf; import org.apache.doris.master.PartitionInfoCollector.PartitionCollectInfo; -import org.apache.doris.thrift.TPartitionVersionInfo; +import org.apache.doris.task.PublishVersionTask.RepublishInfo; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TTablet; import org.apache.doris.thrift.TTabletMetaInfo; @@ -89,7 +89,7 @@ public void tabletReport(long backendId, Map backendTablets, Set tabletFoundInMeta, ListMultimap tabletMigrationMap, Map partitionVersionSyncMap, - Map> transactionsToPublish, + Map> transactionsToPublish, SetMultimap transactionsToClear, ListMultimap tabletRecoveryMap, List tabletToUpdate, diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 4cd4434a73e6c3..66bc4f4dc73f89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -2066,7 +2066,7 @@ public void gsonPostProcess() throws IOException { public abstract void replayModifyProperties(AlterRoutineLoadJobOperationLog log); - public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; + public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo(long txnId) throws UserException; // for ALTER ROUTINE LOAD protected void modifyCommonJobProperties(Map jobProperties) throws UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 0570b356edf133..d9b831362b72c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -1091,7 +1091,7 @@ public Long totalLag() { } @Override - public NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException { + public NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo(long txnId) throws UserException { Expression deleteCondition = getDeleteCondition() != null ? NereidsLoadUtils.parseExpressionSeq( getDeleteCondition().accept(ExprToSqlVisitor.INSTANCE, @@ -1117,7 +1117,7 @@ public NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserExce importColumnDescs.descs.add(new NereidsImportColumnDesc(desc.getColumnName(), expression)); } } - return new NereidsRoutineLoadTaskInfo(execMemLimit, new HashMap<>(jobProperties), maxBatchIntervalS, + return new NereidsRoutineLoadTaskInfo(txnId, execMemLimit, new HashMap<>(jobProperties), maxBatchIntervalS, partitionNamesInfo, mergeType, deleteCondition, sequenceCol, maxFilterRatio, importColumnDescs, precedingFilter, whereExpr, columnSeparator, lineDelimiter, enclose, escape, sendBatchParallelism, loadToSingleTablet, uniqueKeyUpdateMode, partialUpdateNewKeyPolicy, memtableOnSinkNode); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaTaskInfo.java index 322a38151c6add..af9ece10b6b9ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaTaskInfo.java @@ -38,7 +38,6 @@ import org.apache.doris.thrift.TLoadSourceType; import org.apache.doris.thrift.TPipelineFragmentParams; import org.apache.doris.thrift.TPipelineWorkloadGroup; -import org.apache.doris.thrift.TPlanFragment; import org.apache.doris.thrift.TRoutineLoadTask; import org.apache.doris.thrift.TUniqueId; @@ -182,7 +181,7 @@ private TPipelineFragmentParams rePlan(RoutineLoadJob routineLoadJob) throws Use TUniqueId loadId = new TUniqueId(id.getMostSignificantBits(), id.getLeastSignificantBits()); // plan for each task, in case table has change(rollup or schema change) Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(routineLoadJob.getDbId()); - NereidsLoadTaskInfo taskInfo = routineLoadJob.toNereidsRoutineLoadTaskInfo(); + NereidsLoadTaskInfo taskInfo = routineLoadJob.toNereidsRoutineLoadTaskInfo(txnId); taskInfo.setTimeout((int) (this.timeoutMs / 1000)); NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, (OlapTable) db.getTableOrMetaException(routineLoadJob.getTableId(), @@ -190,8 +189,6 @@ private TPipelineFragmentParams rePlan(RoutineLoadJob routineLoadJob) throws Use TPipelineFragmentParams tExecPlanFragmentParams = routineLoadJob.plan(planner, loadId, txnId); StreamLoadHandler.assignAdaptiveRandomBucket(tExecPlanFragmentParams, beId, routineLoadJob.getDbFullName(), routineLoadJob.getTableName()); - TPlanFragment tPlanFragment = tExecPlanFragmentParams.getFragment(); - tPlanFragment.getOutputSink().getOlapTableSink().setTxnId(txnId); if (Config.enable_workload_group) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 7cebc3f5165b49..56e591ab204f59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -847,7 +847,7 @@ public boolean hasMoreDataToConsume(UUID taskId, Map shardIdToSe } @Override - public NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException { + public NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo(long txnId) throws UserException { Expression deleteCondition = getDeleteCondition() != null ? NereidsLoadUtils.parseExpressionSeq( getDeleteCondition().accept(ExprToSqlVisitor.INSTANCE, @@ -873,7 +873,7 @@ public NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserExce importColumnDescs.descs.add(new NereidsImportColumnDesc(desc.getColumnName(), expression)); } } - return new NereidsRoutineLoadTaskInfo(execMemLimit, new HashMap<>(jobProperties), maxBatchIntervalS, + return new NereidsRoutineLoadTaskInfo(txnId, execMemLimit, new HashMap<>(jobProperties), maxBatchIntervalS, partitionNamesInfo, mergeType, deleteCondition, sequenceCol, maxFilterRatio, importColumnDescs, precedingFilter, whereExpr, columnSeparator, lineDelimiter, enclose, escape, sendBatchParallelism, loadToSingleTablet, uniqueKeyUpdateMode, partialUpdateNewKeyPolicy, memtableOnSinkNode); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisTaskInfo.java index a646b17cc5b23e..4ff82e7c371ad8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisTaskInfo.java @@ -36,7 +36,6 @@ import org.apache.doris.thrift.TLoadSourceType; import org.apache.doris.thrift.TPipelineFragmentParams; import org.apache.doris.thrift.TPipelineWorkloadGroup; -import org.apache.doris.thrift.TPlanFragment; import org.apache.doris.thrift.TRoutineLoadTask; import org.apache.doris.thrift.TUniqueId; @@ -196,7 +195,7 @@ private TPipelineFragmentParams rePlan(RoutineLoadJob routineLoadJob) throws Use TUniqueId loadId = new TUniqueId(id.getMostSignificantBits(), id.getLeastSignificantBits()); // plan for each task, in case table has change(rollup or schema change) Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(routineLoadJob.getDbId()); - NereidsLoadTaskInfo taskInfo = routineLoadJob.toNereidsRoutineLoadTaskInfo(); + NereidsLoadTaskInfo taskInfo = routineLoadJob.toNereidsRoutineLoadTaskInfo(txnId); taskInfo.setTimeout((int) (this.timeoutMs / 1000)); NereidsStreamLoadPlanner planner = new NereidsStreamLoadPlanner(db, (OlapTable) db.getTableOrMetaException(routineLoadJob.getTableId(), @@ -204,8 +203,6 @@ private TPipelineFragmentParams rePlan(RoutineLoadJob routineLoadJob) throws Use TPipelineFragmentParams tExecPlanFragmentParams = routineLoadJob.plan(planner, loadId, txnId); StreamLoadHandler.assignAdaptiveRandomBucket(tExecPlanFragmentParams, beId, routineLoadJob.getDbFullName(), routineLoadJob.getTableName()); - TPlanFragment tPlanFragment = tExecPlanFragmentParams.getFragment(); - tPlanFragment.getOutputSink().getOlapTableSink().setTxnId(txnId); if (Config.enable_workload_group) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index a0c08f4b904bf3..ff238eddd845fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -70,6 +70,7 @@ import org.apache.doris.task.DropReplicaTask; import org.apache.doris.task.MasterTask; import org.apache.doris.task.PublishVersionTask; +import org.apache.doris.task.PublishVersionTask.RepublishInfo; import org.apache.doris.task.PushCooldownConfTask; import org.apache.doris.task.PushIndexPolicyTask; import org.apache.doris.task.PushStoragePolicyTask; @@ -93,6 +94,7 @@ import org.apache.doris.thrift.TTabletMetaInfo; import org.apache.doris.thrift.TTabletRole; import org.apache.doris.thrift.TTaskType; +import org.apache.doris.transaction.ChangelogWriteMapping; import com.google.common.base.Preconditions; import com.google.common.collect.LinkedHashMultimap; @@ -610,7 +612,7 @@ public void tabletReport(long backendId, Map backendTablets, Map partitionVersionSyncMap = Maps.newConcurrentMap(); // dbid -> txn id -> [partition info] - Map> transactionsToPublish = Maps.newHashMap(); + Map> transactionsToPublish = Maps.newHashMap(); SetMultimap transactionsToClear = LinkedHashMultimap.create(); // db id -> tablet id @@ -1316,14 +1318,22 @@ static void filterChangelogTabletMigration( } private static void handleRepublishVersionInfo( - Map> transactionsToPublish, long backendId) { + Map> transactionsToPublish, long backendId) { AgentBatchTask batchTask = new AgentBatchTask(); long createPublishVersionTaskTime = System.currentTimeMillis(); for (Long dbId : transactionsToPublish.keySet()) { - SetMultimap map = transactionsToPublish.get(dbId); + SetMultimap map = transactionsToPublish.get(dbId); for (long txnId : map.keySet()) { + List partitions = new ArrayList<>(); + Map mappings = map.get(txnId).iterator().next().getMappings(); + for (RepublishInfo info : map.get(txnId)) { + Preconditions.checkState(mappings.equals(info.getMappings()), + "Conflicting changelog snapshots for transaction %s", txnId); + partitions.add(info.getPartitionVersionInfo()); + } PublishVersionTask task = new PublishVersionTask(backendId, txnId, dbId, - Lists.newArrayList(map.get(txnId)), createPublishVersionTaskTime); + partitions, createPublishVersionTaskTime); + task.setChangelogWriteMappings(mappings); batchTask.addTask(task); // add to AgentTaskQueue for handling finish report. AgentTaskQueue.addTask(task); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 35e271b703a255..4dafc83f1ec6fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -3209,32 +3209,28 @@ private void preserveChangelogSemanticSlots(OlapScanNode scanNode, Set r } } - if (!wrapper.getOriginTable().getChangelogConfig().getNeedHistoricalValue()) { - // Changelog schemas without historical values have no before-image columns, so there - // are no before-image storage dependencies to preserve. - return; - } + Map slotByUid = scanSlots.stream() + .filter(slot -> slot.getColumn() != null) + .collect(Collectors.toMap(slot -> slot.getColumn().getUniqueId(), slot -> slot)); boolean preserveCompleteRow = scanType == TChangelogScanType.MIN_DELTA; - for (SlotDescriptor slot : scanSlots) { - Column column = slot.getColumn(); - if (column == null || column.isKey() || isChangelogInternalColumn(column) - || (!preserveCompleteRow && !requiredSlotIds.contains(slot.getId()))) { + for (Pair pair : wrapper.getChangelogReadColumnPairs()) { + SlotDescriptor current = slotByUid.get(pair.first.getUniqueId()); + SlotDescriptor before = slotByUid.get(pair.second.getUniqueId()); + // A preceding lazy-materialization projection may already have removed an + // unrelated DETAIL pair. MIN_DELTA must retain every pair at each pruning pass. + if (!preserveCompleteRow && current == null && before == null) { continue; } - preserveStorageSlot(slot, requiredSlotIds); - preserveStorageSlot(slotByName.get(Column.generateBeforeColName(column.getName())), - requiredSlotIds); + Preconditions.checkState(current != null && before != null, + "Changelog semantic slots must exist before physical pruning"); + if (preserveCompleteRow || requiredSlotIds.contains(current.getId()) + || requiredSlotIds.contains(before.getId())) { + preserveStorageSlot(current, requiredSlotIds); + preserveStorageSlot(before, requiredSlotIds); + } } } - private boolean isChangelogInternalColumn(Column column) { - String columnName = column.getName(); - return Column.isChangelogBeforeColumn(columnName) - || columnName.equals(Column.CHANGELOG_LSN_COL) - || columnName.equals(Column.CHANGELOG_OP_COL) - || columnName.equals(Column.CHANGELOG_TSO_COL); - } - /** * When rows share a key the sequence column decides the winner, so {@code SELECT v1 FROM * mor_table} without it can hand back the losing row's value. Key-only projections are exempt: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java index 93a075c35f82b0..ec2b834cc139fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.AggregateFunction; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.HdfsResource; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -34,6 +35,7 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.info.PartitionNamesInfo; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; @@ -331,6 +333,10 @@ public Void visitLogicalOlapTableSink(LogicalOlapTableSink logic taskInfo.getSendBatchParallelism(), taskInfo.isLoadToSingleTablet(), taskInfo.isStrictMode(), timeout, uniquekeyUpdateMode, uniquekeyUpdateNewRowPolicy, partialUpdateInputColumns); + if (!Config.isCloudMode() && !(loadPlanInfo.olapTableSink instanceof GroupCommitBlockSink)) { + Env.getCurrentGlobalTransactionMgr().registerChangelogWriteMappings(dbId, taskInfo.getTxnId(), + loadPlanInfo.olapTableSink.getChangelogWriteMappings()); + } } catch (UserException e) { throw new AnalysisException(e.getMessage(), e.getCause()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java index ef159cfb6f494a..e594100c5e1e3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsRoutineLoadTaskInfo.java @@ -71,16 +71,20 @@ public class NereidsRoutineLoadTaskInfo implements NereidsLoadTaskInfo { protected boolean memtableOnSinkNode; protected int timeoutSec; + private final long txnId; + /** * NereidsRoutineLoadTaskInfo */ - public NereidsRoutineLoadTaskInfo(long execMemLimit, Map jobProperties, long maxBatchIntervalS, - PartitionNamesInfo partitions, LoadTask.MergeType mergeType, Expression deleteCondition, + public NereidsRoutineLoadTaskInfo(long txnId, long execMemLimit, Map jobProperties, + long maxBatchIntervalS, PartitionNamesInfo partitions, LoadTask.MergeType mergeType, + Expression deleteCondition, String sequenceCol, double maxFilterRatio, NereidsImportColumnDescs columnDescs, Expression precedingFilter, Expression whereExpr, Separator columnSeparator, Separator lineDelimiter, byte enclose, byte escape, int sendBatchParallelism, boolean loadToSingleTablet, TUniqueKeyUpdateMode uniqueKeyUpdateMode, TPartialUpdateNewRowPolicy partialUpdateNewKeyPolicy, boolean memtableOnSinkNode) { + this.txnId = txnId; this.execMemLimit = execMemLimit; this.jobProperties = jobProperties; this.maxBatchIntervalS = maxBatchIntervalS; @@ -111,7 +115,7 @@ public boolean getNegative() { @Override public long getTxnId() { - return -1L; + return txnId; } public int calTimeoutSec() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java index 70387f50619885..ab130e267eefbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java @@ -41,6 +41,7 @@ import org.apache.doris.planner.DataSink; import org.apache.doris.planner.DataStreamSink; import org.apache.doris.planner.ExchangeNode; +import org.apache.doris.planner.GroupCommitBlockSink; import org.apache.doris.planner.MultiCastDataSink; import org.apache.doris.planner.OlapTableSink; import org.apache.doris.planner.PlanFragment; @@ -158,6 +159,10 @@ public void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink phys false, isStrictMode, timeout, olapInsertCtx); + if (!Config.isCloudMode() && !(olapTableSink instanceof GroupCommitBlockSink)) { + Env.getCurrentGlobalTransactionMgr().registerChangelogWriteMappings(dbId, txnId, + olapTableSink.getChangelogWriteMappings()); + } // set schema and partition info for tablet id shuffle exchange if (fragment.getPlanRoot() instanceof ExchangeNode diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/RemoteOlapInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/RemoteOlapInsertExecutor.java index 511a0a5bd4e836..7e4e5949ca06c5 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/RemoteOlapInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/RemoteOlapInsertExecutor.java @@ -50,12 +50,16 @@ import org.apache.doris.thrift.TPartitionType; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.transaction.BeginTransactionException; +import org.apache.doris.transaction.ChangelogWriteMapping; import org.apache.doris.transaction.TransactionStatus; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Map; + /** * Remote executor for Doris Catalog remote insert. * Local insert implementation remains in {@link OlapInsertExecutor}. @@ -65,9 +69,10 @@ * defined in {@link AbstractInsertExecutor}. */ public class RemoteOlapInsertExecutor extends OlapInsertExecutor { - private static final Logger LOG = LogManager.getLogger(RemoteOlapInsertExecutor.class); + private Map changelogWriteMappings = ImmutableMap.of(); + public RemoteOlapInsertExecutor(ConnectContext ctx, RemoteOlapTable table, String labelName, org.apache.doris.nereids.NereidsPlanner planner, java.util.Optional insertCtx, boolean emptyInsert, @@ -141,6 +146,7 @@ public void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink phys false, isStrictMode, timeout, olapInsertCtx); + changelogWriteMappings = remoteOlapTableSink.getChangelogWriteMappings(); if (fragment.getPlanRoot() instanceof ExchangeNode && fragment.getDataPartition().getType() == TPartitionType.OLAP_TABLE_SINK_HASH_PARTITIONED) { @@ -195,6 +201,7 @@ protected void onComplete() throws UserException { TCommitRemoteTxnRequest request = new TCommitRemoteTxnRequest(); request.setTxnId(txnId); + request.setChangelogColumnMappings(ChangelogWriteMapping.toThrift(changelogWriteMappings)); request.setCatalog(database.getCatalog().getName()); request.setDb(database.getFullName()); request.setTbl(table.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 97f446d05199ee..af75f2124083b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapChangelogTableWrapper; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.stream.BaseTableStream; @@ -52,9 +53,11 @@ import org.apache.doris.nereids.trees.plans.algebra.OlapScan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; +import org.apache.doris.planner.OlapScanNode; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.rpc.RpcException; +import org.apache.doris.thrift.TChangelogScanType; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -1358,7 +1361,14 @@ protected boolean hasSameScanState(LogicalCatalogRelation other) { @Override public boolean supportPruneNestedColumn() { - return true; + if (!scanParams.isPresent() || !scanParams.get().incrementalRead()) { + return true; + } + OlapTable source = table instanceof OlapChangelogTableWrapper + ? ((OlapChangelogTableWrapper) table).getOriginTable() : getTable(); + // MIN_DELTA compares whole values. DETAIL swaps current/before values and therefore + // needs matching materialized types; APPEND_ONLY can still prune individual fields. + return OlapScanNode.parseChangelogScanType(scanParams.get(), source) == TChangelogScanType.APPEND_ONLY; } public Optional getScanParams() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index a975f8f2bf3e5d..259bd6c979a89f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -81,6 +81,8 @@ import org.apache.doris.resource.Tag; import org.apache.doris.resource.computegroup.ComputeGroup; import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TChangelogReadColumnMappings; +import org.apache.doris.thrift.TChangelogReadColumnPair; import org.apache.doris.thrift.TChangelogScanType; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TExplainLevel; @@ -1436,6 +1438,7 @@ protected void toThrift(TPlanNode msg) { msg.olap_scan_node.setIndexesDesc(indexDesc); if (olapTable instanceof OlapChangelogTableWrapper) { msg.olap_scan_node.setReadChangelog(true); + msg.olap_scan_node.setChangelogColumnMappings(buildChangelogReadColumnMappings()); } if (selectedIndexId != -1) { msg.olap_scan_node.setSchemaVersion(olapTable.getIndexSchemaVersion(selectedIndexId)); @@ -2021,6 +2024,32 @@ public static long parseChangeTimestamp(String ts) { throw new ParseException("Invalid timestamp:" + ts); } + private TChangelogReadColumnMappings buildChangelogReadColumnMappings() { + OlapChangelogTableWrapper wrapper = (OlapChangelogTableWrapper) olapTable; + TChangelogScanType scanType = parseChangelogScanType(scanParams, wrapper.getOriginTable()); + List pairs = new ArrayList<>(); + if (scanType == TChangelogScanType.DETAIL || scanType == TChangelogScanType.MIN_DELTA) { + Map slotsByUid = desc.getSlots().stream() + .filter(slot -> slot.getColumn() != null) + .collect(Collectors.toMap(slot -> slot.getColumn().getUniqueId(), slot -> slot)); + for (Pair columns : wrapper.getChangelogReadColumnPairs()) { + SlotDescriptor current = slotsByUid.get(columns.first.getUniqueId()); + SlotDescriptor before = slotsByUid.get(columns.second.getUniqueId()); + if (scanType == TChangelogScanType.MIN_DELTA || current != null || before != null) { + Preconditions.checkState(current != null && before != null, + "Incomplete changelog pair in final scan tuple: %s", columns.first.getName()); + TChangelogReadColumnPair pair = new TChangelogReadColumnPair(); + pair.setCurrentSlotId(current.getId().asInt()); + pair.setBeforeSlotId(before.getId().asInt()); + pairs.add(pair); + } + } + } + TChangelogReadColumnMappings mappings = new TChangelogReadColumnMappings(); + mappings.setPairs(pairs); + return mappings; + } + public static TChangelogScanType parseChangelogScanType(TableScanParams scanParams, OlapTable olapTable) throws ParseException { if (scanParams == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 154beaac8b5e11..689bd2d5f04c9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -87,9 +87,11 @@ import org.apache.doris.thrift.TTabletLocation; import org.apache.doris.thrift.TUniqueId; import org.apache.doris.thrift.TUniqueKeyUpdateMode; +import org.apache.doris.transaction.ChangelogWriteMapping; import com.google.common.base.Preconditions; import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import com.google.common.collect.Range; @@ -130,6 +132,8 @@ public class OlapTableSink extends DataSink { private List partitionExprs; private Map syncMvWhereClauses; + private Map changelogWriteMappings = ImmutableMap.of(); + private boolean initialized; private TOlapTableSchemaParam tOlapTableSchemaParam; private TOlapTablePartitionParam tOlapTablePartitionParam; private TOlapTableLocationParam tOlapTableLocationParam; @@ -152,6 +156,7 @@ public OlapTableSink(OlapTable dstTable, TupleDescriptor tupleDescriptor, List getChangelogWriteMappings() { + Preconditions.checkState(initialized, "OlapTableSink initialization has not completed"); + return changelogWriteMappings; } public TOlapTableSchemaParam getOlapTableSchemaParam() { @@ -401,6 +413,7 @@ protected TDataSink toThrift() { } private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) throws AnalysisException { + Map mappings = new HashMap<>(); TOlapTableSchemaParam schemaParam = new TOlapTableSchemaParam(); schemaParam.setDbId(dbId); schemaParam.setTableId(table.getId()); @@ -442,6 +455,13 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) throws An TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, indexMeta.getSchemaHash()); indexSchema.setChangelogIndexId(indexMeta.getChangelogIndexId()); + if (indexMeta.getChangelogIndexId() > 0) { + MaterializedIndexMeta changelogMeta = table.getIndexMetaByIndexId(indexMeta.getChangelogIndexId()); + ChangelogWriteMapping mapping = ChangelogWriteMapping.fromColumns(indexMeta.getSchema(true), + changelogMeta.getSchema(true), table.getChangelogConfig().getNeedHistoricalValue()); + mappings.put(pair.getKey(), mapping); + indexSchema.setChangelogColumnMappings(mapping.toThrift()); + } Expr whereClause = indexMeta.getWhereClause(); if (whereClause != null) { Expr expr = syncMvWhereClauses.getOrDefault(pair.getKey(), null); @@ -473,6 +493,7 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) throws An setPartialUpdateInfoForParam(schemaParam, table, uniqueKeyUpdateMode); schemaParam.setInvertedIndexFileStorageFormat(table.getInvertedIndexFileStorageFormat()); + changelogWriteMappings = ImmutableMap.copyOf(mappings); return schemaParam; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 50873fb01d6da3..632a4d4dfec5ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -328,6 +328,7 @@ import org.apache.doris.thrift.TWaitingTxnStatusRequest; import org.apache.doris.thrift.TWaitingTxnStatusResult; import org.apache.doris.transaction.BeginTransactionException; +import org.apache.doris.transaction.ChangelogWriteMapping; import org.apache.doris.transaction.SubTransactionState; import org.apache.doris.transaction.TabletCommitInfo; import org.apache.doris.transaction.Transaction; @@ -2434,6 +2435,20 @@ private boolean commitRemoteTxnImpl(TCommitRemoteTxnRequest request) throws User } TableIf tableIf = db.getTableOrMetaException(request.getTbl(), TableType.OLAP); OlapTable table = (OlapTable) tableIf; + try { + Map mappings = + ChangelogWriteMapping.fromThrift(request.getChangelogColumnMappings()); + if (!Config.isCloudMode()) { + TransactionState state = Env.getCurrentGlobalTransactionMgr().getTransactionState( + db.getId(), request.getTxnId()); + if (state == null) { + throw new UserException("Missing remote transaction " + request.getTxnId()); + } + state.installRemoteChangelogWriteMappings(mappings); + } + } catch (IllegalArgumentException | IllegalStateException e) { + throw new UserException("Invalid remote changelog snapshot: " + e.getMessage(), e); + } return Env.getCurrentGlobalTransactionMgr().commitAndPublishTransaction( db, Lists.newArrayList(table), request.getTxnId(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/PublishVersionTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/PublishVersionTask.java index 63a82a9f885398..84073b7d4f6262 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/PublishVersionTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/PublishVersionTask.java @@ -20,7 +20,9 @@ import org.apache.doris.thrift.TPartitionVersionInfo; import org.apache.doris.thrift.TPublishVersionRequest; import org.apache.doris.thrift.TTaskType; +import org.apache.doris.transaction.ChangelogWriteMapping; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import lombok.Getter; @@ -30,11 +32,46 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; public class PublishVersionTask extends AgentTask { private static final Logger LOG = LogManager.getLogger(PublishVersionTask.class); + /** Partition and immutable write snapshot captured together while processing a tablet report. */ + public static final class RepublishInfo { + private final TPartitionVersionInfo partitionVersionInfo; + private final Map mappings; + + public RepublishInfo(TPartitionVersionInfo partitionVersionInfo, Map mappings) { + this.partitionVersionInfo = partitionVersionInfo.deepCopy(); + this.mappings = ImmutableMap.copyOf(mappings); + } + + public TPartitionVersionInfo getPartitionVersionInfo() { + return partitionVersionInfo.deepCopy(); + } + + public Map getMappings() { + return mappings; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof RepublishInfo)) { + return false; + } + RepublishInfo info = (RepublishInfo) other; + return partitionVersionInfo.equals(info.partitionVersionInfo) && mappings.equals(info.mappings); + } + + @Override + public int hashCode() { + return Objects.hash(partitionVersionInfo, mappings); + } + } + + private Map changelogWriteMappings = ImmutableMap.of(); private long transactionId; @Getter private List partitionVersionInfos; @@ -69,9 +106,14 @@ public TPublishVersionRequest toThrift() { TPublishVersionRequest publishVersionRequest = new TPublishVersionRequest(transactionId, partitionVersionInfos); publishVersionRequest.setBaseTabletIds(baseTabletsIds); + publishVersionRequest.setChangelogColumnMappings(ChangelogWriteMapping.toThrift(changelogWriteMappings)); return publishVersionRequest; } + public void setChangelogWriteMappings(Map mappings) { + changelogWriteMappings = ImmutableMap.copyOf(mappings); + } + public void setBaseTabletsIds(Set rollupTabletIds) { this.baseTabletsIds = rollupTabletIds; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/ChangelogWriteMapping.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/ChangelogWriteMapping.java new file mode 100644 index 00000000000000..3a5f00adf2e703 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/ChangelogWriteMapping.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.transaction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.thrift.TChangelogIndexWriteSnapshot; +import org.apache.doris.thrift.TChangelogWriteColumnMapping; +import org.apache.doris.thrift.TChangelogWriteColumnMappings; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; + +/** Immutable UID relationships captured from the schema used to plan a write. */ +public final class ChangelogWriteMapping { + private static final class Entry { + @SerializedName("source") + private final int source; + @SerializedName("current") + private final int current; + @SerializedName("before") + private final Integer before; + + private Entry(int source, int current, Integer before) { + this.source = source; + this.current = current; + this.before = before; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Entry)) { + return false; + } + Entry entry = (Entry) other; + return source == entry.source && current == entry.current && Objects.equals(before, entry.before); + } + + @Override + public int hashCode() { + return Objects.hash(source, current, before); + } + } + + @SerializedName("historical") + private final boolean needHistoricalValue; + @SerializedName("entries") + private final List entries; + + private ChangelogWriteMapping(boolean historical, List entries) { + Set sources = new HashSet<>(); + Set targets = new HashSet<>(); + for (Entry entry : entries) { + Preconditions.checkArgument(sources.add(entry.source), "Duplicate source UID: %s", entry.source); + Preconditions.checkArgument(targets.add(entry.current), "Duplicate target UID: %s", entry.current); + if (entry.before != null) { + Preconditions.checkArgument(historical, "Before UID requires historical values"); + Preconditions.checkArgument(targets.add(entry.before), "Duplicate target UID: %s", entry.before); + } + } + List sorted = new ArrayList<>(entries); + sorted.sort(Comparator.comparingInt(entry -> entry.source)); + this.entries = ImmutableList.copyOf(sorted); + this.needHistoricalValue = historical; + } + + public static ChangelogWriteMapping fromColumns(List sourceColumns, List changelogColumns, + boolean historical) { + Map targets = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (Column column : changelogColumns) { + Preconditions.checkState(targets.put(column.getNonShadowName(), column) == null, + "Duplicate changelog column: %s", column.getName()); + } + List entries = new ArrayList<>(); + for (Column source : sourceColumns) { + if (!source.isVisible() && !source.isKey()) { + continue; + } + Column current = targets.get(source.getNonShadowName()); + Preconditions.checkState(current != null, "Missing current column: %s", source.getName()); + Integer beforeUid = null; + if (historical && !source.isKey()) { + Column before = targets.get(Column.generateBeforeColName(source.getNonShadowName())); + Preconditions.checkState(before != null, "Missing before column: %s", source.getName()); + beforeUid = before.getUniqueId(); + } + entries.add(new Entry(source.getUniqueId(), current.getUniqueId(), beforeUid)); + } + return new ChangelogWriteMapping(historical, entries); + } + + public static ChangelogWriteMapping fromThrift(TChangelogWriteColumnMappings mappings) { + Preconditions.checkArgument(mappings != null && mappings.isSetEntries() + && mappings.isSetNeedHistoricalValue(), "Changelog mapping requires entries and historical flag"); + List entries = new ArrayList<>(); + for (TChangelogWriteColumnMapping entry : mappings.getEntries()) { + Preconditions.checkArgument(entry != null && entry.isSetSourceColumnUniqueId() + && entry.isSetCurrentColumnUniqueId(), "Changelog entry requires source and current UID"); + entries.add(new Entry(entry.getSourceColumnUniqueId(), entry.getCurrentColumnUniqueId(), + entry.isSetBeforeColumnUniqueId() ? entry.getBeforeColumnUniqueId() : null)); + } + return new ChangelogWriteMapping(mappings.isNeedHistoricalValue(), entries); + } + + public TChangelogWriteColumnMappings toThrift() { + List result = new ArrayList<>(); + for (Entry entry : entries) { + TChangelogWriteColumnMapping mapping = new TChangelogWriteColumnMapping() + .setSourceColumnUniqueId(entry.source).setCurrentColumnUniqueId(entry.current); + if (entry.before != null) { + mapping.setBeforeColumnUniqueId(entry.before); + } + result.add(mapping); + } + return new TChangelogWriteColumnMappings().setEntries(result).setNeedHistoricalValue(needHistoricalValue); + } + + public static List toThrift(Map mappings) { + List result = new ArrayList<>(); + new TreeMap<>(mappings).forEach((indexId, mapping) -> result.add(new TChangelogIndexWriteSnapshot() + .setSourceIndexId(indexId).setMappings(mapping.toThrift()))); + return result; + } + + public static Map fromThrift(List snapshots) { + Preconditions.checkArgument(snapshots != null, "Missing changelog index snapshots"); + Map result = new TreeMap<>(); + for (TChangelogIndexWriteSnapshot snapshot : snapshots) { + Preconditions.checkArgument(snapshot != null && snapshot.isSetSourceIndexId() && snapshot.isSetMappings(), + "Changelog snapshot requires source index and mappings"); + Preconditions.checkArgument(result.put(snapshot.getSourceIndexId(), fromThrift(snapshot.getMappings())) + == null, "Duplicate changelog source index: %s", snapshot.getSourceIndexId()); + } + return ImmutableMap.copyOf(result); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ChangelogWriteMapping)) { + return false; + } + ChangelogWriteMapping mapping = (ChangelogWriteMapping) other; + return needHistoricalValue == mapping.needHistoricalValue && entries.equals(mapping.entries); + } + + @Override + public int hashCode() { + return Objects.hash(needHistoricalValue, entries); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 2d37d7b0bde912..9f71eff8f3a1a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -1940,9 +1940,16 @@ protected void unprotectUpdateInMemoryState(TransactionState transactionState, b finalStatusTransactionStateDequeLong.add(transactionState); } if (transactionState.getSubTxnIds() != null) { - LOG.info("clean sub transactions for txn_id={}, sub_txn_ids={}", transactionState.getTransactionId(), - transactionState.getSubTxnIds()); - cleanSubTransactions(transactionState.getTransactionId()); + if (transactionState.getTransactionStatus() == TransactionStatus.VISIBLE + && transactionState.hasChangelogWriteMappings()) { + // Lagging replicas report the actual child id after visibility, including after FE replay. + // Keep these aliases until the retained parent transaction is removed. + for (Long subTxnId : transactionState.getSubTxnIds()) { + addSubTransaction(transactionState.getTransactionId(), subTxnId); + } + } else { + cleanSubTransactions(transactionState.getTransactionId()); + } } } if (isReplay) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index 1e9a388e141621..49f85c2ae628ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -187,6 +187,22 @@ public TWaitingTxnStatusResult getWaitingTxnStatus(TWaitingTxnStatusRequest requ public TransactionState getTransactionState(long dbId, long transactionId); + default void registerChangelogWriteMappings(long dbId, long writeTxnId, Map mappings) + throws UserException { + if (mappings.isEmpty()) { + return; + } + TransactionState state = getTransactionState(dbId, writeTxnId); + if (state == null) { + throw new UserException("Cannot register changelog mappings for missing transaction " + writeTxnId); + } + try { + state.registerChangelogWriteMappings(writeTxnId, mappings); + } catch (IllegalStateException e) { + throw new UserException(e.getMessage(), e); + } + } + public Long getTransactionIdByLabel(Long dbId, String label, List statusList) throws UserException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java index fcbd6705ac248e..009b16a5c085d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java @@ -437,6 +437,7 @@ private void addPublishVersionTask(Set publishBackends, long transactionId transactionState.getDbId(), partitionVersionInfos, createPublishVersionTaskTime); + task.setChangelogWriteMappings(transactionState.getChangelogWriteMappings(transactionId)); task.setBaseTabletsIds(beIdToBaseTabletIds.getOrDefault(backendId, Collections.emptySet())); // add to AgentTaskQueue for handling finish report. // not check return value, because the add will success diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java index bee6ec5e0598f2..32310f1d17b924 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java @@ -27,6 +27,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.metric.MetricRepo; +import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.task.PublishVersionTask; import org.apache.doris.thrift.TUniqueId; @@ -34,6 +35,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -58,7 +60,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -public class TransactionState implements Writable { +public class TransactionState implements Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(TransactionState.class); // compare the TransactionState by txn id, desc @@ -336,6 +338,76 @@ public SchemaInfo(OlapTable olapTable) { @SerializedName(value = "sui") private List streamUpdateInfos; + // Actual write transaction id (including subtransactions) -> source index -> write-time snapshot. + @SerializedName("changelogWriteMappings") + private volatile Map> changelogWriteMappings = ImmutableMap.of(); + @SerializedName("remoteChangelogMappingsInstalled") + private boolean remoteChangelogMappingsInstalled; + + public boolean hasChangelogWriteMappings() { + return changelogWriteMappings.values().stream().anyMatch(mappings -> !mappings.isEmpty()); + } + + public synchronized Map getChangelogWriteMappings(long writeTxnId) { + return changelogWriteMappings.getOrDefault(writeTxnId, ImmutableMap.of()); + } + + public synchronized void registerChangelogWriteMappings(long writeTxnId, + Map mappings) { + checkChangelogRegistrationStatus(); + Map existing = getChangelogWriteMappings(writeTxnId); + Map updated = new TreeMap<>(existing); + for (Map.Entry entry : mappings.entrySet()) { + ChangelogWriteMapping previous = existing.get(entry.getKey()); + if (previous != null) { + Preconditions.checkState(previous.equals(entry.getValue()), + "Changelog mapping changed for transaction %s, source index %s", writeTxnId, entry.getKey()); + } else { + Preconditions.checkState(transactionStatus == TransactionStatus.PREPARE + && !remoteChangelogMappingsInstalled, + "Changelog snapshots frozen for transaction %s (%s)", writeTxnId, transactionStatus); + updated.put(entry.getKey(), entry.getValue()); + } + } + if (!updated.equals(existing)) { + Map> all = new TreeMap<>(changelogWriteMappings); + all.put(writeTxnId, ImmutableMap.copyOf(updated)); + changelogWriteMappings = ImmutableMap.copyOf(all); + } + } + + public synchronized void installRemoteChangelogWriteMappings(Map mappings) { + checkChangelogRegistrationStatus(); + Map snapshot = ImmutableMap.copyOf(mappings); + if (remoteChangelogMappingsInstalled) { + Preconditions.checkState(getChangelogWriteMappings(transactionId).equals(snapshot), + "Remote changelog snapshot changed for transaction %s", transactionId); + return; + } + Preconditions.checkState(transactionStatus == TransactionStatus.PREPARE, + "Cannot install remote changelog snapshot for frozen transaction %s (%s)", + transactionId, transactionStatus); + Preconditions.checkState(changelogWriteMappings.isEmpty(), + "Remote transaction %s already has local changelog snapshots", transactionId); + changelogWriteMappings = ImmutableMap.of(transactionId, snapshot); + remoteChangelogMappingsInstalled = true; + } + + private void checkChangelogRegistrationStatus() { + Preconditions.checkState(transactionStatus == TransactionStatus.PREPARE + || transactionStatus == TransactionStatus.PRECOMMITTED + || transactionStatus == TransactionStatus.COMMITTED + || transactionStatus == TransactionStatus.VISIBLE, + "Cannot register changelog mappings for transaction %s (%s)", transactionId, transactionStatus); + } + + @Override + public void gsonPostProcess() throws IOException { + Map> snapshots = new TreeMap<>(); + changelogWriteMappings.forEach((txnId, mappings) -> snapshots.put(txnId, ImmutableMap.copyOf(mappings))); + changelogWriteMappings = ImmutableMap.copyOf(snapshots); + } + public TransactionState() { this.dbId = -1; this.tableIdList = Lists.newArrayList(); @@ -524,7 +596,7 @@ public void setLastPublishLogTime(long lastPublishLogTime) { this.lastPublishLogTime = lastPublishLogTime; } - public void setTransactionStatus(TransactionStatus transactionStatus) { + public synchronized void setTransactionStatus(TransactionStatus transactionStatus) { // status changed this.preStatus = this.transactionStatus; this.transactionStatus = transactionStatus; diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableChangelogSchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableChangelogSchemaTest.java index e5fedabd84dbdf..f3b33cb790aa8a 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableChangelogSchemaTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableChangelogSchemaTest.java @@ -19,6 +19,7 @@ import org.apache.doris.binlog.BinlogTestUtils; import org.apache.doris.thrift.TStorageType; +import org.apache.doris.transaction.ChangelogWriteMapping; import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; @@ -61,6 +62,21 @@ private static OlapTable newTestTable(ChangelogConfig changelogConfig, List source = table.getBaseSchema(true); + Assertions.assertTrue(source.get(0).getUniqueId() >= 0); + Assertions.assertNotEquals(source.get(0).getUniqueId(), source.get(1).getUniqueId()); + ChangelogWriteMapping mapping = ChangelogWriteMapping.fromColumns(source, + table.getChangelogIndexMeta().getSchema(true), true); + Assertions.assertEquals(2, mapping.toThrift().getEntriesSize()); + Assertions.assertTrue(mapping.toThrift().getEntries().get(1).isSetBeforeColumnUniqueId()); + } + @Test public void testChangelogSchemaOnEnable() { OlapTable tableWithoutBefore = newTestTable(BinlogTestUtils.newTestChangelogConfig(true, false)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index 000c5a375fe9d8..a1681d17883526 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.SortInfo; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; @@ -54,6 +55,7 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; @@ -93,6 +95,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; @@ -131,6 +134,10 @@ protected void runBeforeAll() throws Exception { + "'enable_unique_key_merge_on_write' = 'true'," + "'changelog.enable' = 'true'," + "'changelog.need_historical_value' = 'true');"); + createTable("create table test_db.changelog_without_history_t(k int, v int) " + + "unique key(k) distributed by hash(k) buckets 1 " + + "properties('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true'," + + "'changelog.enable' = 'true', 'changelog.need_historical_value' = 'false');"); createTable("create table test_db.sequence_scan_schema_t(\n" + "k1 int, k2 int, v1 int, v2 int)\n" + "unique key(k1, k2)\n" @@ -265,6 +272,13 @@ public void testChangelogPhysicalScanSchemaDependencies() throws Exception { .collect(Collectors.toList())); TPlanNode thriftScanNode = scanNode.treeToThrift().getNodes().get(0); + Assertions.assertTrue(thriftScanNode.olap_scan_node.isSetChangelogColumnMappings()); + Assertions.assertEquals(2, thriftScanNode.olap_scan_node.getChangelogColumnMappings().getPairsSize()); + Map slotNames = scanNode.getTupleDesc().getSlots().stream() + .collect(Collectors.toMap(slot -> slot.getId().asInt(), slot -> slot.getColumn().getName())); + thriftScanNode.olap_scan_node.getChangelogColumnMappings().getPairs().forEach(pair -> + Assertions.assertEquals(Column.generateBeforeColName(slotNames.get(pair.getCurrentSlotId())), + slotNames.get(pair.getBeforeSlotId()))); Set dependencyUniqueIds = scanNode.getTupleDesc().getSlots().stream() .filter(slot -> !slot.getColumn().getName().equals("v1")) .map(slot -> slot.getColumn().getUniqueId()) @@ -273,6 +287,20 @@ public void testChangelogPhysicalScanSchemaDependencies() throws Exception { .noneMatch(thriftScanNode.olap_scan_node.getOutputColumnUniqueIds()::contains)); } + @Test + public void testChangelogMappingsFollowFinalTupleAfterReordering() throws Exception { + OlapScanNode scan = getFirstOlapScanNode("select v1 from test_db.changelog_scan_schema_t" + + "@incr(\"incrementType\" = \"MIN_DELTA\")"); + Collections.reverse(scan.getTupleDesc().getSlots()); + Map namesById = scan.getTupleDesc().getSlots().stream() + .collect(Collectors.toMap(slot -> slot.getId().asInt(), slot -> slot.getColumn().getName())); + TPlanNode thrift = scan.treeToThrift().getNodes().get(0); + Assertions.assertEquals(2, thrift.olap_scan_node.getChangelogColumnMappings().getPairsSize()); + thrift.olap_scan_node.getChangelogColumnMappings().getPairs().forEach(pair -> + Assertions.assertEquals(Column.generateBeforeColName(namesById.get(pair.getCurrentSlotId())), + namesById.get(pair.getBeforeSlotId()))); + } + @Test public void testAppendOnlyChangelogPhysicalScanSchemaDependencies() throws Exception { OlapScanNode scanNode = getFirstOlapScanNode( @@ -287,6 +315,32 @@ public void testAppendOnlyChangelogPhysicalScanSchemaDependencies() throws Excep Assertions.assertTrue(scanNode.getExtraKeyColumnSlotIds().isEmpty()); } + @Test + public void testDetailWithoutHistoryHasExplicitEmptyMappings() throws Exception { + OlapScanNode scan = getFirstOlapScanNode("select v from test_db.changelog_without_history_t" + + "@incr(\"incrementType\" = \"DETAIL\")"); + TPlanNode thrift = scan.treeToThrift().getNodes().get(0); + Assertions.assertTrue(thrift.olap_scan_node.isSetChangelogColumnMappings()); + Assertions.assertTrue(thrift.olap_scan_node.getChangelogColumnMappings().isSetPairs()); + Assertions.assertTrue(thrift.olap_scan_node.getChangelogColumnMappings().getPairs().isEmpty()); + } + + @Test + public void testChangelogNestedPruningDependsOnReadMode() throws Exception { + Database database = (Database) Env.getCurrentInternalCatalog().getDbOrMetaException("test_db"); + OlapTable table = (OlapTable) database.getTableOrMetaException("changelog_scan_schema_t"); + LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), + new OlapChangelogTableWrapper(table)); + Assertions.assertTrue(scan.withTableScanParams(new TableScanParams(TableScanParams.INCREMENTAL_READ, + Collections.singletonMap(OlapScanNode.OLAP_INCREMENT_TYPE, "APPEND_ONLY"), + Collections.emptyList())).supportPruneNestedColumn()); + for (String mode : ImmutableList.of("DETAIL", "MIN_DELTA")) { + Assertions.assertFalse(scan.withTableScanParams(new TableScanParams(TableScanParams.INCREMENTAL_READ, + Collections.singletonMap(OlapScanNode.OLAP_INCREMENT_TYPE, mode), + Collections.emptyList())).supportPruneNestedColumn()); + } + } + @Test public void testMergeSequencePhysicalScanSchemaDependencies() throws Exception { OlapScanNode sequenceScan = getFirstOlapScanNode( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java index 044817079d5cec..dba4843624a69b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/load/VariantLoadParseInjectionTest.java @@ -149,7 +149,7 @@ private LoadFixture createStreamFixture(InputShape shape) throws Exception { } private LoadFixture createRoutineFixture(InputShape shape) throws Exception { - NereidsRoutineLoadTaskInfo task = new NereidsRoutineLoadTaskInfo(1024L, new HashMap<>(), 10L, + NereidsRoutineLoadTaskInfo task = new NereidsRoutineLoadTaskInfo(3L, 1024L, new HashMap<>(), 10L, null, LoadTask.MergeType.APPEND, null, null, 0.0, columnDescs(shape), null, null, null, null, (byte) 0, (byte) 0, 1, false, TUniqueKeyUpdateMode.UPSERT, TPartialUpdateNewRowPolicy.APPEND, false); diff --git a/fe/fe-core/src/test/java/org/apache/doris/transaction/ChangelogWriteMappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/transaction/ChangelogWriteMappingTest.java new file mode 100644 index 00000000000000..e99e3b0548e534 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/transaction/ChangelogWriteMappingTest.java @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.transaction; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.LocalTabletInvertedIndex; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TabletMeta; +import org.apache.doris.common.UserException; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.master.ReportHandler; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.task.AgentTaskExecutor; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.task.PublishVersionTask; +import org.apache.doris.thrift.TChangelogIndexWriteSnapshot; +import org.apache.doris.thrift.TChangelogWriteColumnMapping; +import org.apache.doris.thrift.TChangelogWriteColumnMappings; +import org.apache.doris.thrift.TPartitionVersionInfo; +import org.apache.doris.thrift.TPublishVersionRequest; +import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.thrift.TTaskType; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.SetMultimap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class ChangelogWriteMappingTest { + private TChangelogWriteColumnMapping entry(int source, int current) { + return new TChangelogWriteColumnMapping().setSourceColumnUniqueId(source).setCurrentColumnUniqueId(current); + } + + private ChangelogWriteMapping mapping(boolean historical, TChangelogWriteColumnMapping... entries) { + return ChangelogWriteMapping.fromThrift(new TChangelogWriteColumnMappings() + .setNeedHistoricalValue(historical).setEntries(Arrays.asList(entries))); + } + + @Test + void columnRelationshipsUseUidsDespiteReorderedTargets() { + Column key = new Column("key", PrimitiveType.INT); + key.setIsKey(true); + key.setUniqueId(3); + Column value = new Column("value", PrimitiveType.INT); + value.setUniqueId(7); + Column currentKey = Column.generateChangelogKeyColumn(key); + currentKey.setUniqueId(31); + Column currentValue = Column.generateAfterValueColumn(value); + currentValue.setUniqueId(71); + Column before = Column.generateBeforeValueColumn(value); + before.setUniqueId(72); + ChangelogWriteMapping mapping = ChangelogWriteMapping.fromColumns(Arrays.asList(key, value), + Arrays.asList(before, currentKey, currentValue), true); + List entries = mapping.toThrift().getEntries(); + Assertions.assertEquals(3, entries.get(0).getSourceColumnUniqueId()); + Assertions.assertEquals(31, entries.get(0).getCurrentColumnUniqueId()); + Assertions.assertFalse(entries.get(0).isSetBeforeColumnUniqueId()); + Assertions.assertEquals(7, entries.get(1).getSourceColumnUniqueId()); + Assertions.assertEquals(71, entries.get(1).getCurrentColumnUniqueId()); + Assertions.assertEquals(72, entries.get(1).getBeforeColumnUniqueId()); + Assertions.assertThrows(IllegalStateException.class, () -> ChangelogWriteMapping.fromColumns( + Arrays.asList(key, value), Arrays.asList(currentKey, currentValue), true)); + } + + @Test + void indexSnapshotRoundTripRejectsMissingAndDuplicateIndexes() { + ChangelogWriteMapping mapping = mapping(false, entry(1, 1)); + Map snapshots = ImmutableMap.of(7L, mapping, 2L, mapping); + List wire = ChangelogWriteMapping.toThrift(snapshots); + Assertions.assertEquals(2L, wire.get(0).getSourceIndexId()); + Assertions.assertEquals(snapshots, ChangelogWriteMapping.fromThrift(wire)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> ChangelogWriteMapping.fromThrift(Arrays.asList(wire.get(0), wire.get(0)))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> ChangelogWriteMapping.fromThrift(Collections.singletonList(new TChangelogIndexWriteSnapshot()))); + Assertions.assertEquals(Collections.emptyMap(), ChangelogWriteMapping.fromThrift(Collections.emptyList())); + } + + @Test + void publishKeepsActualChildSnapshotAfterReplayAndReportCapture() { + TransactionState state = new TransactionState(); + state.registerChangelogWriteMappings(101, ImmutableMap.of(5L, mapping(true, entry(1, 11)))); + state.registerChangelogWriteMappings(102, ImmutableMap.of(5L, mapping(false, entry(1, 21)))); + state.setTransactionStatus(TransactionStatus.VISIBLE); + TransactionState replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(state), TransactionState.class); + PublishVersionTask.RepublishInfo report = new PublishVersionTask.RepublishInfo( + new TPartitionVersionInfo(40, 2, 0), replay.getChangelogWriteMappings(102)); + PublishVersionTask task = new PublishVersionTask(10, 102, 1, + Collections.singletonList(report.getPartitionVersionInfo()), 0); + task.setChangelogWriteMappings(report.getMappings()); + TPublishVersionRequest request = task.toThrift(); + Assertions.assertEquals(102, request.getTransactionId()); + Assertions.assertEquals(5, request.getChangelogColumnMappings().get(0).getSourceIndexId()); + Assertions.assertEquals(21, request.getChangelogColumnMappings().get(0).getMappings() + .getEntries().get(0).getCurrentColumnUniqueId()); + Assertions.assertFalse(request.getChangelogColumnMappings().get(0).getMappings().isNeedHistoricalValue()); + request.getChangelogColumnMappings().clear(); + Assertions.assertEquals(1, task.toThrift().getChangelogColumnMappingsSize()); + } + + @Test + void tabletReportRepublishCapturesChildMappingAndPartitionTogether() { + TransactionState state = new TransactionState(); + state.setSubTxnIds(Arrays.asList(101L, 102L)); + state.registerChangelogWriteMappings(101, ImmutableMap.of(5L, mapping(true, entry(1, 11)))); + state.registerChangelogWriteMappings(102, ImmutableMap.of(5L, mapping(false, entry(1, 21)))); + TableCommitInfo commit = new TableCommitInfo(20); + commit.addPartitionCommitInfo(new PartitionCommitInfo(30, "", 40, 0, false)); + state.getSubTxnIdToTableCommitInfo().put(102L, commit); + state.setTransactionStatus(TransactionStatus.VISIBLE); + Map> reports = new HashMap<>(); + Deencapsulation.invoke(new LocalTabletInvertedIndex(), "publishPartition", state, 102L, + new TabletMeta(-1, 20, 30, 5, 0, TStorageMedium.HDD, false), 30L, reports); + try (MockedStatic executor = Mockito.mockStatic(AgentTaskExecutor.class)) { + Deencapsulation.invoke(ReportHandler.class, "handleRepublishVersionInfo", reports, 9001L); + PublishVersionTask task = (PublishVersionTask) AgentTaskQueue.getTask(9001, TTaskType.PUBLISH_VERSION, 102); + TPublishVersionRequest request = task.toThrift(); + Assertions.assertEquals(40, request.getPartitionVersionInfos().get(0).getVersion()); + Assertions.assertEquals(21, request.getChangelogColumnMappings().get(0).getMappings() + .getEntries().get(0).getCurrentColumnUniqueId()); + Assertions.assertFalse(request.getChangelogColumnMappings().get(0).getMappings().isNeedHistoricalValue()); + } finally { + AgentTaskQueue.removeTask(9001, TTaskType.PUBLISH_VERSION, 102); + } + } + + @Test + void registrationBoundaryRejectsMissingAndAbortedTransactions() throws Exception { + GlobalTransactionMgrIface manager = Mockito.mock(GlobalTransactionMgrIface.class, Mockito.CALLS_REAL_METHODS); + Map snapshots = ImmutableMap.of(5L, mapping(false, entry(1, 10))); + Assertions.assertThrows(UserException.class, () -> manager.registerChangelogWriteMappings(1, 100, snapshots)); + TransactionState state = new TransactionState(); + Mockito.when(manager.getTransactionState(1, 100)).thenReturn(state); + manager.registerChangelogWriteMappings(1, 100, snapshots); + Assertions.assertEquals(snapshots, state.getChangelogWriteMappings(100)); + state.setTransactionStatus(TransactionStatus.ABORTED); + Assertions.assertThrows(UserException.class, () -> manager.registerChangelogWriteMappings(1, 100, snapshots)); + Assertions.assertDoesNotThrow(() -> manager.registerChangelogWriteMappings(1, 999, Collections.emptyMap())); + } + + @Test + void installationDetachesMutableCallerMaps() { + Map input = new HashMap<>(); + input.put(5L, mapping(false, entry(1, 10))); + TransactionState state = new TransactionState(); + state.registerChangelogWriteMappings(100, input); + input.clear(); + Assertions.assertEquals(Collections.singleton(5L), state.getChangelogWriteMappings(100).keySet()); + } + + @Test + void canonicalOrderAndKeyOnlyHistoricalRoundTrip() { + ChangelogWriteMapping first = mapping(true, entry(2, 20), entry(1, 10)); + ChangelogWriteMapping second = mapping(true, entry(1, 10), entry(2, 20)); + Assertions.assertEquals(first, second); + Assertions.assertEquals(first.hashCode(), second.hashCode()); + Assertions.assertEquals(first, ChangelogWriteMapping.fromThrift(first.toThrift())); + Assertions.assertTrue(first.toThrift().isNeedHistoricalValue()); + Assertions.assertEquals(1, first.toThrift().getEntries().get(0).getSourceColumnUniqueId()); + first.toThrift().getEntries().clear(); + Assertions.assertEquals(2, first.toThrift().getEntriesSize()); + } + + @Test + void malformedMappingsAreRejected() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> ChangelogWriteMapping.fromThrift(new TChangelogWriteColumnMappings())); + Assertions.assertThrows(IllegalArgumentException.class, () -> mapping(false, entry(1, 10), entry(1, 11))); + Assertions.assertThrows(IllegalArgumentException.class, () -> mapping(false, entry(1, 10), entry(2, 10))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> mapping(false, entry(1, 10).setBeforeColumnUniqueId(11))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> mapping(true, entry(1, 10).setBeforeColumnUniqueId(10))); + Assertions.assertDoesNotThrow(() -> mapping(false, entry(1, 1))); + } + + @Test + void precommitFreezesAtomicSnapshotsAndReplayPreservesThem() throws Exception { + TransactionState state = new TransactionState(); + ChangelogWriteMapping mapping = mapping(true, entry(1, 10)); + state.registerChangelogWriteMappings(100, ImmutableMap.of(5L, mapping)); + Assertions.assertThrows(IllegalStateException.class, () -> state.registerChangelogWriteMappings(100, + ImmutableMap.of(6L, mapping, 5L, mapping(false, entry(1, 10))))); + Assertions.assertEquals(Collections.singleton(5L), state.getChangelogWriteMappings(100).keySet()); + state.setTransactionStatus(TransactionStatus.PRECOMMITTED); + TransactionState replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(state), TransactionState.class); + replay.registerChangelogWriteMappings(100, ImmutableMap.of(5L, mapping)); + Assertions.assertThrows(IllegalStateException.class, + () -> replay.registerChangelogWriteMappings(100, ImmutableMap.of(6L, mapping))); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> replay.getChangelogWriteMappings(100).clear()); + Assertions.assertEquals(mapping, replay.getChangelogWriteMappings(100).get(5L)); + } + + @Test + void remoteEmptyIsCompleteAndCannotBeInstalledAfterFreeze() { + TransactionState state = new TransactionState(); + state.installRemoteChangelogWriteMappings(Collections.emptyMap()); + state.setTransactionStatus(TransactionStatus.COMMITTED); + state.installRemoteChangelogWriteMappings(Collections.emptyMap()); + TransactionState replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(state), TransactionState.class); + replay.installRemoteChangelogWriteMappings(Collections.emptyMap()); + Assertions.assertThrows(IllegalStateException.class, () -> state.installRemoteChangelogWriteMappings( + ImmutableMap.of(5L, mapping(false, entry(1, 10))))); + TransactionState frozen = new TransactionState(); + frozen.setTransactionStatus(TransactionStatus.PRECOMMITTED); + Assertions.assertThrows(IllegalStateException.class, + () -> frozen.installRemoteChangelogWriteMappings(Collections.emptyMap())); + frozen.setTransactionStatus(TransactionStatus.UNKNOWN); + Assertions.assertThrows(IllegalStateException.class, + () -> frozen.registerChangelogWriteMappings(100, Collections.emptyMap())); + frozen.setTransactionStatus(TransactionStatus.ABORTED); + Assertions.assertThrows(IllegalStateException.class, + () -> frozen.registerChangelogWriteMappings(100, Collections.emptyMap())); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/transaction/DatabaseTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/transaction/DatabaseTransactionMgrTest.java index 3d41243955cae0..dc133a060ecee0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/transaction/DatabaseTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/transaction/DatabaseTransactionMgrTest.java @@ -50,9 +50,13 @@ import org.apache.doris.meta.MetaContext; import org.apache.doris.mtmv.ivm.IvmInfo; import org.apache.doris.mysql.authenticate.TestLogAppender; +import org.apache.doris.persist.BatchRemoveTransactionsOperationV2; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.resource.Tag; import org.apache.doris.system.Backend; import org.apache.doris.task.PublishVersionTask; +import org.apache.doris.thrift.TChangelogWriteColumnMapping; +import org.apache.doris.thrift.TChangelogWriteColumnMappings; import org.apache.doris.thrift.TPartitionVersionInfo; import org.apache.doris.thrift.TUniqueId; import org.apache.doris.transaction.GlobalTransactionMgrTest.SubTransactionInfo; @@ -707,7 +711,7 @@ public void testSubTransaction() throws UserException { long subTransactionId3 = transactionState6.getSubTxnIds().get(2); TransactionState subTransactionState = masterTransMgr.getTransactionState(CatalogTestUtil.testDbId1, subTransactionId3); - Assertions.assertEquals(null, subTransactionState); // finished txn will remove sub txn map + Assertions.assertNull(subTransactionState); // Ordinary finished transactions remove sub transaction aliases. // test show transaction state command /*List> singleTranInfos = masterDbTransMgr.getSingleTranInfo(CatalogTestUtil.testDbId1, subTransactionId3); @@ -763,6 +767,28 @@ public void testSubTransaction() throws UserException { Assertions.assertNull(masterDbTransMgr.getTransactionState(subTransactionId3)); } + @Test + public void testMappedVisibleChildAliasSurvivesReplayUntilEviction() throws Exception { + DatabaseTransactionMgr dbMgr = masterTransMgr.getDatabaseTransactionMgr(CatalogTestUtil.testDbId1); + TransactionState state = new TransactionState(CatalogTestUtil.testDbId1, Collections.emptyList(), + 987650, "mapped_visible", null, LoadJobSourceType.FRONTEND, transactionSource, -1, 1000); + state.setSubTxnIds(Collections.singletonList(987651L)); + ChangelogWriteMapping mapping = ChangelogWriteMapping.fromThrift(new TChangelogWriteColumnMappings() + .setNeedHistoricalValue(true).setEntries(Collections.singletonList( + new TChangelogWriteColumnMapping().setSourceColumnUniqueId(1).setCurrentColumnUniqueId(2)))); + state.registerChangelogWriteMappings(987651, Collections.singletonMap(5L, mapping)); + state.setTransactionStatus(TransactionStatus.VISIBLE); + TransactionState replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(state), TransactionState.class); + dbMgr.unprotectUpsertTransactionState(replay, true); + Assertions.assertSame(replay, dbMgr.getTransactionState(987651L)); + Assertions.assertEquals(mapping, dbMgr.getTransactionState(987651L).getChangelogWriteMappings(987651).get(5L)); + // Replay the current batch-eviction record, which also removes older retained transactions. + dbMgr.replayBatchRemoveTransaction(new BatchRemoveTransactionsOperationV2( + CatalogTestUtil.testDbId1, -1, replay.getTransactionId())); + Assertions.assertNull(dbMgr.getTransactionState(987650L)); + Assertions.assertNull(dbMgr.getTransactionState(987651L)); + } + @Test public void testRemoveExpiredTxnsWithSubTxn() throws UserException { addSubTransaction(); @@ -1026,6 +1052,45 @@ private void checkAbortDuringCommitTso(List tables, TransactionState stat } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testPrecommitChangelogSnapshotSurvivesReplay(boolean logOutsideLock) throws Exception { + boolean previous = Config.enable_txn_log_outside_lock; + try { + Config.enable_txn_log_outside_lock = logOutsideLock; + FakeEnv.setEnv(masterEnv); + Database db = masterEnv.getInternalCatalog().getDbOrMetaException(CatalogTestUtil.testDbId1); + OlapTable table = (OlapTable) db.getTableOrMetaException(CatalogTestUtil.testTableId1); + long txnId = masterTransMgr.beginTransaction(db.getId(), Lists.newArrayList(table.getId()), + "precommit_changelog_snapshot", transactionSource, LoadJobSourceType.FRONTEND, + Config.stream_load_default_timeout_second); + ChangelogWriteMapping mapping = ChangelogWriteMapping.fromThrift(new TChangelogWriteColumnMappings() + .setNeedHistoricalValue(true).setEntries(Collections.singletonList( + new TChangelogWriteColumnMapping().setSourceColumnUniqueId(1).setCurrentColumnUniqueId(2)))); + Map snapshots = Collections.singletonMap(table.getBaseIndexId(), mapping); + masterTransMgr.registerChangelogWriteMappings(db.getId(), txnId, snapshots); + masterTransMgr.preCommitTransaction2PC(db, Lists.newArrayList(table), txnId, + GlobalTransactionMgrTest.generateTabletCommitInfos(CatalogTestUtil.testTabletId1, allBackends), + 1000, null); + TransactionState persisted = fakeEditLog.getTransaction(txnId); + Assertions.assertEquals(TransactionStatus.PRECOMMITTED, persisted.getTransactionStatus()); + TransactionState replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(persisted), TransactionState.class); + Assertions.assertEquals(snapshots, replay.getChangelogWriteMappings(txnId)); + replay.registerChangelogWriteMappings(txnId, snapshots); + Assertions.assertThrows(IllegalStateException.class, () -> replay.registerChangelogWriteMappings(txnId, + Collections.singletonMap(table.getBaseIndexId() + 100, mapping))); + FakeEnv.setEnv(slaveEnv); + slaveTransMgr.replayUpsertTransactionState(replay); + Database replayDb = slaveEnv.getInternalCatalog().getDbOrMetaException(db.getId()); + Table replayTable = replayDb.getTableOrMetaException(table.getId()); + slaveTransMgr.commitTransaction2PC(replayDb, Lists.newArrayList(replayTable), txnId, 1000); + Assertions.assertEquals(TransactionStatus.COMMITTED, replay.getTransactionStatus()); + Assertions.assertEquals(snapshots, replay.getChangelogWriteMappings(txnId)); + } finally { + Config.enable_txn_log_outside_lock = previous; + } + } + @ParameterizedTest @CsvSource({"normal,false,false", "normal,true,false", "2pc,false,false", "2pc,true,false", "subtxn,false,false", "subtxn,true,false", "normal,false,true", "normal,true,true", diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index 2c00ac2f82b5e5..8cc33aaa1496cc 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -105,6 +105,7 @@ message POlapTableIndexSchema { repeated TabletIndexPB indexes_desc = 5; // ATTN(changelog-rename): formerly "row_binlog_id" optional int64 changelog_index_id = 6; + optional PChangelogWriteColumnMappings changelog_column_mappings = 7; }; message POlapTableSchemaParam { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 95886a271e1753..659e990527b7c5 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -80,6 +80,17 @@ message TsoRangePB { optional int64 end_tso = 2 [default = -1]; } +message PChangelogWriteColumnMapping { + optional int32 source_column_unique_id = 1; + optional int32 current_column_unique_id = 2; + optional int32 before_column_unique_id = 3; +} + +message PChangelogWriteColumnMappings { + repeated PChangelogWriteColumnMapping entries = 1; + optional bool need_historical_value = 2; +} + // ATTN: When adding or deleting fields, please update `message RowsetMetaCloudPB` // simultaneously and modify the conversion function in the be/src/cloud/pb_convert.{h,cpp}. message RowsetMetaPB { @@ -851,6 +862,7 @@ message PendingPublishInfoPB { optional int64 partition_id = 1; optional int64 transaction_id = 2; optional int64 commit_tso = 3; + optional PChangelogWriteColumnMappings changelog_column_mappings = 4; } message RowsetBinlogMetasPB { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 49907f4a4ba475..60ffc784ab658c 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -505,6 +505,8 @@ struct TPublishVersionRequest { 3: optional bool strict_mode = false // for delta rows statistics to exclude rollup tablets 4: optional set base_tablet_ids + // Snapshots from this write transaction, keyed by each entry's source index. + 8: optional list changelog_column_mappings } struct TVisibleVersionReq { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 6ea733a707d051..0f97d14fe4a9b2 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -348,6 +348,32 @@ struct TOlapTableIndex { 7: optional list column_unique_ids } +// Stable write-time identities; positions are resolved against the transaction schemas in BE. +struct TChangelogWriteColumnMapping { + 1: optional i32 source_column_unique_id + 2: optional i32 current_column_unique_id + 3: optional i32 before_column_unique_id +} + +struct TChangelogWriteColumnMappings { + 1: optional list entries + 2: optional bool need_historical_value +} + +struct TChangelogIndexWriteSnapshot { + 1: optional i64 source_index_id + 2: optional TChangelogWriteColumnMappings mappings +} + +struct TChangelogReadColumnPair { + 1: optional Types.TSlotId current_slot_id + 2: optional Types.TSlotId before_slot_id +} + +struct TChangelogReadColumnMappings { + 1: optional list pairs +} + struct TOlapTableIndexSchema { 1: required i64 id 2: required list columns @@ -357,6 +383,7 @@ struct TOlapTableIndexSchema { 6: optional Exprs.TExpr where_clause // ATTN(changelog-rename): formerly "row_binlog_id" 7: optional i64 changelog_index_id + 10: optional TChangelogWriteColumnMappings changelog_column_mappings } struct TOlapTableSchemaParam { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 7d96a9e7415bce..d4559e7f97079b 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1848,6 +1848,8 @@ struct TCommitRemoteTxnRequest { 7: optional i64 txn_id 8: optional list commit_infos 9: optional i64 insert_visible_timeout_ms + // Complete writer snapshot; new callers explicitly send an empty list without changelog. + 13: optional list changelog_column_mappings } struct TCommitRemoteTxnResult { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 1cb49705c5bd4d..cb782f74d75221 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -1136,6 +1136,7 @@ struct TOlapScanNode { // equals the sole HASH distribution key, and all scan ranges carry bucket metadata. // BE may prune only final inclusive point ranges; absent means disabled. 29: optional bool enable_scan_key_bucket_prune + 31: optional Descriptors.TChangelogReadColumnMappings changelog_column_mappings } struct TEqJoinCondition {