diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp index 5cf8c7df01da7b..26fbbec7daa07e 100644 --- a/be/src/agent/heartbeat_server.cpp +++ b/be/src/agent/heartbeat_server.cpp @@ -97,6 +97,10 @@ void HeartbeatServer::heartbeat(THeartbeatResult& heartbeat_result, heartbeat_result.backend_info.__set_be_mem(MemInfo::physical_mem()); // Reader enum support cannot be inferred from a release version during rolling upgrades. heartbeat_result.backend_info.__set_supports_paimon_rust_reader(true); + // The PK-vector (ANN) protocol is a separate negotiation: an older + // rust-capable BE would reject an empty paimon_split + vector payload as + // "missing paimon_split", so only BEs carrying the vector protocol report it. + heartbeat_result.backend_info.__set_supports_paimon_rust_vector_reader(true); } watch.stop(); if (watch.elapsed_time() > 1000L * 1000L * 1000L) { diff --git a/be/src/format_v2/table/paimon_rust_predicate_converter.h b/be/src/format_v2/table/paimon_rust_predicate_converter.h index 76c76037b51a97..be861707ce4b8e 100644 --- a/be/src/format_v2/table/paimon_rust_predicate_converter.h +++ b/be/src/format_v2/table/paimon_rust_predicate_converter.h @@ -42,6 +42,27 @@ namespace doris {} // namespace doris namespace doris { +// Reader-produced PK-vector search distance column, as Doris names it. Wire +// contract shared with FE (PaimonVectorSearch.SEARCH_DISTANCE_COLUMN). It is never +// a projectable table column and never a pushable predicate column, so both the +// projection builder and the predicate converter must exclude it. +// +// It holds the metric-native DISTANCE, not paimon-rust's score: paimon-rust emits +// an Arrow field named kPaimonRustScoreField carrying a higher-is-better score, and +// the rust table reader inverts that transform in place while filling the block so +// the value matches what l2_distance_approximate / inner_product_approximate are +// defined to return. Lower-is-better for l2. +// +// The two names deliberately differ, since the two values mean different things -- +// and keeping them distinct leaves __paimon_search_score free should a later +// version want to expose the raw score as its own column. +constexpr const char* kPaimonSearchDistanceColumn = "__paimon_search_score_to_dis"; + +// The Arrow field name paimon-rust auto-appends to a vector-search batch (see the +// bindings' materialize_search_result), so it cannot be changed from here; the +// reader maps it onto kPaimonSearchDistanceColumn when filling the block. +constexpr const char* kPaimonRustScoreField = "__paimon_search_score"; + // Converts Doris push-down conjuncts into a paimon-rust filter predicate. // // This mirrors PaimonPredicateConverter (the paimon-cpp variant) but targets the diff --git a/be/src/format_v2/table/paimon_rust_table_reader.cpp b/be/src/format_v2/table/paimon_rust_table_reader.cpp index 731508957bbf4f..472f34a20260f2 100644 --- a/be/src/format_v2/table/paimon_rust_table_reader.cpp +++ b/be/src/format_v2/table/paimon_rust_table_reader.cpp @@ -18,19 +18,27 @@ #include "format_v2/table/paimon_rust_table_reader.h" #include +#include +#include +#include #include #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/record_batch.h" #include "arrow/result.h" +#include "arrow/type.h" +#include "common/compiler_util.h" #include "common/logging.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" +#include "exec/common/endian.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" #include "format_v2/column_mapper.h" @@ -70,6 +78,15 @@ PAIMON_OWNED(read_builder, paimon_read_builder_free); PAIMON_OWNED(plan, paimon_plan_free); PAIMON_OWNED(table_read, paimon_table_read_free); PAIMON_OWNED(record_batch_reader, paimon_record_batch_reader_free); +PAIMON_OWNED(vector_search_builder, paimon_vector_search_builder_free); +// Local handles of the eager Top-K vector-search chain (paimon-rust #822): they only +// need to live until the arrow reader is created, so unlike the table/reader handles +// above they are function locals of _open_vector_split_reader rather than members +// of PaimonHandles. +PAIMON_OWNED(bucket_vector_search_split, paimon_bucket_vector_search_split_free); +PAIMON_OWNED(vector_scan, paimon_vector_scan_free); +PAIMON_OWNED(vector_plan, paimon_vector_plan_free); +PAIMON_OWNED(vector_read, paimon_vector_read_free); PAIMON_OWNED(error, paimon_error_free); #undef PAIMON_OWNED @@ -346,14 +363,26 @@ Status PaimonRustTableReader::_validate_rust_split(const TFileRangeDesc& range) "mismatch"); } const auto& params = range.table_format_params.paimon_params; - if (!params.__isset.paimon_split || params.paimon_split.empty()) { + if (params.__isset.reader_type && params.reader_type != TPaimonReaderType::PAIMON_RUST) { return Status::InternalError( - "missing paimon_split for paimon rust reader, possibly caused by FE/BE protocol " + "invalid reader_type for paimon rust reader, possibly caused by FE/BE protocol " "mismatch"); } - if (params.__isset.reader_type && params.reader_type != TPaimonReaderType::PAIMON_RUST) { + // Vector (ANN) mode: the payload is the serialized BucketVectorSearchSplit, and + // paimon_split is deliberately an empty string (FE avoids serializing the same + // DataSplit twice on the wire). An empty-but-set paimon_split therefore also + // implies vector mode: reporting "missing paimon_split" for such a split would + // point the reader at the wrong field when the real gap is the vector payload. + if (params.__isset.vector_payload || + (params.__isset.paimon_split && params.paimon_split.empty())) { + if (!params.__isset.vector_payload || params.vector_payload.bytes.empty()) { + return Status::InternalError( + "missing vector_payload for paimon rust vector reader, possibly caused by " + "FE/BE protocol mismatch"); + } + } else if (!params.__isset.paimon_split || params.paimon_split.empty()) { return Status::InternalError( - "invalid reader_type for paimon rust reader, possibly caused by FE/BE protocol " + "missing paimon_split for paimon rust reader, possibly caused by FE/BE protocol " "mismatch"); } if (!_resolve_table_path(range).has_value()) { @@ -376,20 +405,93 @@ Status PaimonRustTableReader::_validate_rust_split(const TFileRangeDesc& range) return Status::OK(); } -Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) { - // 1. Decode the FE-planned split first so we fail fast (and without any - // filesystem IO) when it is missing or malformed. - std::string split_bytes; - RETURN_IF_ERROR(_decode_split_bytes(&split_bytes)); +bool PaimonRustTableReader::_is_vector_mode(const TFileRangeDesc& range) { + return range.__isset.table_format_params && range.table_format_params.__isset.paimon_params && + range.table_format_params.paimon_params.__isset.vector_payload; +} + +Status PaimonRustTableReader::_validate_external_search_request() const { + if (!_scan_params || !_scan_params->__isset.external_search_request) { + return Status::InvalidArgument( + "paimon-rust vector search missing external_search_request in scan params"); + } + const auto& request = _scan_params->external_search_request; + if (request.__isset.schema_version && request.schema_version != 1) { + return Status::NotSupported("unsupported external search schema version: {}", + request.schema_version); + } + if (!request.__isset.search_query || !request.search_query.__isset.vector_search) { + return Status::InvalidArgument( + "paimon-rust vector search requires a vector query in external_search_request"); + } - // 2. Resolve identifier + table_path + FE-supplied TableSchema JSON. + const auto& vector = request.search_query.vector_search; + // column + if (!vector.__isset.column || vector.column.empty() || + vector.column.find('\0') != std::string::npos) { + return Status::InvalidArgument("paimon-rust vector search requires a non-empty column"); + } + // query_vector (TSearchVector) + if (!vector.__isset.query_vector) { + return Status::InvalidArgument("paimon-rust vector search requires a query vector"); + } + const auto& query_vector = vector.query_vector; + if (!query_vector.__isset.element_type || !query_vector.__isset.dimension || + !query_vector.__isset.values) { + return Status::InvalidArgument( + "paimon-rust query vector requires element_type, dimension, and values"); + } + if (query_vector.dimension <= 0) { + return Status::InvalidArgument("paimon-rust query vector dimension must be positive: {}", + query_vector.dimension); + } + if (query_vector.element_type != TVectorElementType::FLOAT32) { + return Status::NotSupported( + "paimon-rust vector search only supports FLOAT32 query vector, got {}", + static_cast(query_vector.element_type)); + } + const auto dimension = static_cast(query_vector.dimension); + if (query_vector.values.size() != dimension * sizeof(float)) { + return Status::InvalidArgument( + "paimon-rust query vector byte size {} does not match dimension {} * {}", + query_vector.values.size(), dimension, sizeof(float)); + } + // top_k / offset + if (!vector.__isset.top_k || vector.top_k < 0) { + return Status::InvalidArgument("paimon-rust vector search top_k must be non-negative"); + } + if (!vector.__isset.offset || vector.offset < 0) { + return Status::InvalidArgument("paimon-rust vector search offset must be non-negative"); + } + // metric is required (NOT advisory): BE uses it to undo the score transform. + // paimon-rust only supports L2 and DOT_PRODUCT; parse_score_transform rejects + // anything else, so validate __isset here and let parse_score_transform handle + // the value. + if (!vector.__isset.metric) { + return Status::InvalidArgument("paimon-rust vector search requires a metric"); + } + // paimon_options (optional): only validate the map itself is present if set. + if (request.__isset.paimon_options && request.paimon_options.__isset.options) { + for (const auto& kv : request.paimon_options.options) { + if (kv.first.find('\0') != std::string::npos || + kv.second.find('\0') != std::string::npos) { + return Status::InvalidArgument( + "paimon-rust vector search options contain an embedded NUL byte"); + } + } + } + return Status::OK(); +} + +Status PaimonRustTableReader::_open_paimon_table(const TFileRangeDesc& range) { + // Resolve identifier + table_path + FE-supplied TableSchema JSON. auto table_path = _resolve_table_path(range).value(); auto db_name = _resolve_db_name(range).value(); auto table_name = _resolve_table_name(range).value(); auto schema_json = _resolve_table_schema_json(range).value(); auto branch_opt = _resolve_branch(range); - // 3. Assemble storage options: FE-supplied paimon options + hadoop_conf + + // Assemble storage options: FE-supplied paimon options + hadoop_conf + // OSS/S3 → AWS_* translations. These feed FileIO only (per // paimon_table_from_schema_json contract); they are NOT merged into the // supplied table schema. @@ -397,42 +499,57 @@ Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) { auto opened_table_key = std::make_tuple(table_path, schema_json, db_name, table_name, branch_opt, options); - if (!_handles || !_handles->table || _opened_table_key != opened_table_key) { + if (_handles && _handles->table && _opened_table_key == opened_table_key) { // A paimon scan reads one table, so the handle is opened at most once per - // distinct identity (e.g. re-created after a close); splits of the same - // table reuse it and only rebuild the read pipeline below. - _close_table(); - _handles = std::make_unique(); - - std::vector c_options; - c_options.reserve(options.size()); - for (const auto& kv : options) { - c_options.push_back(paimon_option {kv.first.c_str(), kv.second.c_str()}); - } - - LOG(INFO) << "paimon-rust opening table via schema json: db=" << db_name - << " table=" << table_name << " path=" << table_path - << " branch=" << (branch_opt.has_value() ? branch_opt.value() : "main") - << " storage_options=[" << format_options(options) << "]"; - - // Build the table directly from the FE-supplied schema JSON. The Rust - // side rejects null / empty branch, so we default to paimon's canonical - // "main" sentinel when FE did not set paimon_branch (i.e. the table is - // on the main branch — matches upstream Identifier.DEFAULT_MAIN_BRANCH). - const std::string& branch_str = branch_opt.has_value() ? branch_opt.value() : "main"; - paimon_result_get_table tbl_res = paimon_table_from_schema_json( - table_path.c_str(), schema_json.c_str(), db_name.c_str(), table_name.c_str(), - branch_str.c_str(), c_options.empty() ? nullptr : c_options.data(), - c_options.size()); - if (tbl_res.error != nullptr) { - return Status::InternalError( - "paimon-rust table_from_schema_json failed: db={} table={} err={}", db_name, - table_name, consume_error(tbl_res.error)); - } - _handles->table.reset(tbl_res.table); - _opened_table_key = std::move(opened_table_key); + // distinct identity; splits of the same table reuse it and only rebuild + // the read pipeline. + return Status::OK(); + } + _close_table(); + _handles = std::make_unique(); + + std::vector c_options; + c_options.reserve(options.size()); + for (const auto& kv : options) { + c_options.push_back(paimon_option {kv.first.c_str(), kv.second.c_str()}); } + LOG(INFO) << "paimon-rust opening table via schema json: db=" << db_name + << " table=" << table_name << " path=" << table_path + << " branch=" << (branch_opt.has_value() ? branch_opt.value() : "main") + << " storage_options=[" << format_options(options) << "]"; + + // Build the table directly from the FE-supplied schema JSON. The Rust + // side rejects null / empty branch, so we default to paimon's canonical + // "main" sentinel when FE did not set paimon_branch (i.e. the table is + // on the main branch — matches upstream Identifier.DEFAULT_MAIN_BRANCH). + const std::string& branch_str = branch_opt.has_value() ? branch_opt.value() : "main"; + paimon_result_get_table tbl_res = paimon_table_from_schema_json( + table_path.c_str(), schema_json.c_str(), db_name.c_str(), table_name.c_str(), + branch_str.c_str(), c_options.empty() ? nullptr : c_options.data(), c_options.size()); + if (tbl_res.error != nullptr) { + return Status::InternalError( + "paimon-rust table_from_schema_json failed: db={} table={} err={}", db_name, + table_name, consume_error(tbl_res.error)); + } + _handles->table.reset(tbl_res.table); + _opened_table_key = std::move(opened_table_key); + return Status::OK(); +} + +Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) { + // Primary-key vector (ANN) search: route to the vector-search read path. + if (_is_vector_mode(range)) { + return _open_vector_split_reader(range); + } + // 1. Decode the FE-planned split first so we fail fast (and without any + // filesystem IO) when it is missing or malformed. + std::string split_bytes; + RETURN_IF_ERROR(_decode_split_bytes(&split_bytes)); + + // 2. Resolve and open (or reuse) the table. + RETURN_IF_ERROR(_open_paimon_table(range)); + // 4. Build the read pipeline: read_builder -> case-insensitive -> projection. // Bound the Rust Arrow allocation itself: wide rows must respect the scanner's // adaptive probe size before they are materialized into a Doris block. @@ -509,6 +626,316 @@ Status PaimonRustTableReader::_open_split_reader(const TFileRangeDesc& range) { return Status::OK(); } +Status PaimonRustTableReader::_open_vector_split_reader(const TFileRangeDesc& range) { + // Validate every __isset / value-range up front (mirrors + // LanceTableReader::_validate_external_search_request). The build path below + // then uses the fields unchecked, matching lance's split of validate-once / + // build-without-recheck. + RETURN_IF_ERROR(_validate_external_search_request()); + const auto& request = _scan_params->external_search_request; + const auto& vector = request.search_query.vector_search; + + // 0. Distance metric. It decides which score transform the reader has to undo + // (see _convert_score_to_distance), so defaulting would silently return wrong + // distances. __isset was checked by _validate_external_search_request; the + // value is rejected here if unsupported. + ScoreTransform transform; + RETURN_IF_ERROR(parse_score_transform(vector.metric, &transform)); + _score_transform = transform; + + // 1. The per-split payload is the serialized BucketVectorSearchSplit (raw + // thrift binary, NOT base64; magic "PKVSPLIT", v1). Grab it first so we fail + // fast before any filesystem IO. + const auto& payload = _current_range.table_format_params.paimon_params.vector_payload; + if (payload.payload_type != TPaimonVectorPayloadType::BUCKET_SPLIT_BYTES) { + return Status::InternalError("paimon-rust vector search unsupported payload type: {}", + static_cast(payload.payload_type)); + } + const std::string& split_bytes = payload.bytes; + if (split_bytes.empty()) { + return Status::InternalError("paimon-rust vector search split payload is empty"); + } + + // 2. Resolve and open (or reuse) the table (same as the normal read path); + // options carry storage credentials to the FileIO. + RETURN_IF_ERROR(_open_paimon_table(range)); + + // 3. Build and configure the vector-search builder. No read.batch-size here, + // unlike the batch path (_open_split_reader): paimon-rust's PK vector readers + // construct their DataFileReaders without with_batch_size, so a + // read.batch-size option — from the schema, t@options or these search + // options — is silently ignored on the vector path, which batches at the + // parquet-rust default. Known difference; the vector stages are + // single-column streamed into bounded Top-K heaps and the output is + // capped by top_k, so no action needed until paimon-rust wires it. + paimon_result_vector_search_builder vb_res = + paimon_table_new_vector_search_builder(_handles->table.get()); + if (vb_res.error != nullptr) { + return Status::InternalError("paimon-rust new vector search builder failed: {}", + consume_error(vb_res.error)); + } + vector_search_builder_ptr vb(vb_res.builder); + + // 3a. Target vector column. + if (paimon_error* err = + paimon_vector_search_builder_with_vector_column(vb.get(), vector.column.c_str())) { + return Status::InternalError("paimon-rust set vector column failed: {}", + consume_error(err)); + } + // 3b. Query vector. The wire form is a TSearchVector carrying a FLOAT32, + // little-endian binary blob; decode it into the f32 array the C API wants. + // v1 only supports FLOAT32 (paimon-rust's index backends operate on f32); any + // other element type was already rejected by validate. Dimension and + // byte-size were validated there too, so decode unchecked here. + const auto& search_vec = vector.query_vector; + const auto dimension = static_cast(search_vec.dimension); + std::vector query_vector(dimension); + const auto* bytes = search_vec.values.data(); + for (size_t i = 0; i < dimension; ++i) { + query_vector[i] = std::bit_cast(LittleEndian::Load32( + reinterpret_cast(bytes + i * sizeof(uint32_t)))); + } + if (paimon_error* err = paimon_vector_search_builder_with_query_vector( + vb.get(), query_vector.data(), query_vector.size())) { + return Status::InternalError("paimon-rust set query vector failed: {}", consume_error(err)); + } + // 3c. Retrieval limit (user_limit + user_offset, folded on FE). A negative + // limit would wrap when cast to uintptr_t, so guard it. + if (vector.top_k < 0) { + return Status::InternalError("paimon-rust vector search limit is negative: {}", + vector.top_k); + } + if (paimon_error* err = paimon_vector_search_builder_with_limit( + vb.get(), static_cast(vector.top_k))) { + return Status::InternalError("paimon-rust set limit failed: {}", consume_error(err)); + } + // 3d. Index/search options (refine_factor, ivf.nprobe, ".metric", ...). + if (request.__isset.paimon_options && request.paimon_options.__isset.options && + !request.paimon_options.options.empty()) { + std::vector c_search_options; + c_search_options.reserve(request.paimon_options.options.size()); + for (const auto& kv : request.paimon_options.options) { + c_search_options.emplace_back(kv.first.c_str(), kv.second.c_str()); + } + if (paimon_error* err = paimon_vector_search_builder_with_options( + vb.get(), c_search_options.data(), c_search_options.size())) { + return Status::InternalError("paimon-rust set search options failed: {}", + consume_error(err)); + } + } + // 3e. Materialized user columns. The score field is always appended by + // paimon-rust itself and must NOT be listed here. Empty projection = only the + // score column (index-only scan). Log the user columns: an empty list is + // legitimate, so it is logged as such rather than skipped. + auto read_columns = _build_read_columns(); + std::vector projection; + projection.reserve(read_columns.size() + 1); + for (const auto& col : read_columns) { + projection.push_back(col.c_str()); + } + projection.push_back(nullptr); + LOG(INFO) << "paimon-rust vector search: projection=[" << join(read_columns, ", ") << "] (" + << read_columns.size() << " column(s), distance column '" + << kPaimonSearchDistanceColumn << "' produced by the reader)"; + if (paimon_error* err = + paimon_vector_search_builder_with_projection(vb.get(), projection.data())) { + return Status::InternalError("paimon-rust set vector projection failed: {}", + consume_error(err)); + } + + // 3f. Optional residual scalar filter from the FE data-half push-down. The + // FE-side DV/merge-on-read interception is intentionally not done (see + // PushDownVectorTopNIntoPaimonScan): the Rust candidate search stage applies + // the filter together with deletion vectors, and a table shape Rust cannot + // handle correctly is Rust's responsibility to reject. + if (!_conjuncts.empty()) { + std::vector names; + std::vector types; + names.reserve(_projected_columns.size()); + types.reserve(_projected_columns.size()); + for (const auto& col : _projected_columns) { + if (col.is_partition_key || col.name == kPaimonSearchDistanceColumn) { + continue; + } + names.push_back(col.name); + types.push_back(col.type); + } + PaimonRustPredicateConverter converter(names, types, _handles->table.get()); + paimon_predicate* predicate = converter.build(_conjuncts); + if (predicate != nullptr) { + // with_filter consumes the predicate on every path. + if (paimon_error* err = paimon_vector_search_builder_with_filter(vb.get(), predicate)) { + return Status::InternalError("paimon-rust set vector filter failed: {}", + consume_error(err)); + } + LOG(INFO) << "paimon-rust vector search: applied residual filter"; + } + } + + // 4. Execute the search over the single FE-planned bucket split, via the + // eager Top-K chain (paimon-rust #822). Search, refine, local Top-K and + // materialization all happen in paimon_vector_read_read; the returned arrow + // stream is best-first and carries the __paimon_search_score column. The + // intermediate handles are locals: the migration contract (see + // doris-vector-search-api-migration.md) guarantees the record batch reader + // is independent once paimon_vector_read_read returns, so they are released + // when this function exits. + // + // split: the bytes only need to live until deserialize returns. + // scan: only needs to live until the plan is constructed. + // builder: only needs to live until scan and read are created. + // read/plan: only need to live until paimon_vector_read_read returns. + paimon_result_bucket_vector_search_split split_result = + paimon_bucket_vector_search_split_deserialize( + reinterpret_cast(split_bytes.data()), split_bytes.size()); + if (split_result.error != nullptr) { + return Status::InternalError("paimon-rust deserialize vector split failed: {}", + consume_error(split_result.error)); + } + bucket_vector_search_split_ptr split(split_result.split); + + paimon_result_vector_scan scan_result = paimon_vector_search_builder_new_scan(vb.get()); + if (scan_result.error != nullptr) { + return Status::InternalError("paimon-rust new vector scan failed: {}", + consume_error(scan_result.error)); + } + vector_scan_ptr scan(scan_result.scan); + + paimon_result_vector_read read_result = paimon_vector_search_builder_new_read(vb.get()); + if (read_result.error != nullptr) { + return Status::InternalError("paimon-rust new vector read failed: {}", + consume_error(read_result.error)); + } + vector_read_ptr read(read_result.read); + + // Must use plan_from_bucket_splits: it plans from the FE-assigned split, + // whereas paimon_vector_scan_plan / execute_read replan from the table and + // would ignore the split this scanner was given. + const paimon_bucket_vector_search_split* splits[] = {split.get()}; + paimon_result_vector_plan plan_result = + paimon_vector_scan_plan_from_bucket_splits(scan.get(), splits, 1); + if (plan_result.error != nullptr) { + return Status::InternalError("paimon-rust build vector plan failed: {}", + consume_error(plan_result.error)); + } + vector_plan_ptr plan(plan_result.plan); + + paimon_result_record_batch_reader rdr_res = paimon_vector_read_read(read.get(), plan.get()); + if (rdr_res.error != nullptr) { + return Status::InternalError("paimon-rust vector search read failed: {}", + consume_error(rdr_res.error)); + } + _handles->reader.reset(rdr_res.reader); + return Status::OK(); +} + +std::string PaimonRustTableReader::arrow_field_to_block_column(const std::string& arrow_name) { + std::string name = to_lower(arrow_name); + if (name == kPaimonRustScoreField) { + return kPaimonSearchDistanceColumn; + } + return name; +} + +Status PaimonRustTableReader::parse_score_transform(TVectorMetric::type metric, + ScoreTransform* transform) { + switch (metric) { + case TVectorMetric::L2: + *transform = ScoreTransform::L2; + return Status::OK(); + case TVectorMetric::DOT_PRODUCT: + *transform = ScoreTransform::InnerProduct; + return Status::OK(); + default: + // Includes COSINE: paimon-rust does define a score for it + // (score = 1 - distance), but v1 does not push cosine down, so reaching here + // means FE and BE disagree. Fail instead of inventing a transform. + return Status::InternalError( + "paimon-rust vector search unsupported distance metric: {} " + "(expected L2 or DOT_PRODUCT)", + static_cast(metric)); + } +} + +void PaimonRustTableReader::_convert_score_to_distance(Block* block, uint32_t block_idx, + size_t row_offset, size_t num_rows, + ScoreTransform transform) { + // Undo what paimon-rust's convert_distance_to_score did, so the column holds + // what the Doris distance function is defined to return. + switch (transform) { + case ScoreTransform::InnerProduct: + // score IS the inner product. Nothing to undo, and no clamping to do + // either: the value is unconstrained by definition (it may be negative). + return; + case ScoreTransform::L2: + break; + } + + auto& column_with_type = block->get_by_position(block_idx); + auto column = IColumn::mutate(std::move(column_with_type.column)); + PaddedPODArray* data = nullptr; + const NullMap* null_map = nullptr; + if (auto* nullable = check_and_get_column(column.get())) { + data = &assert_cast(nullable->get_nested_column()).get_data(); + null_map = &nullable->get_null_map_data(); + } else { + data = &assert_cast(*column).get_data(); + } + + // score = 1/(1 + d) where d is the SQUARED L2 distance, and + // l2_distance_approximate returns the true Euclidean one, so invert both steps: + // d_euclidean = sqrt(1/score - 1). + const size_t end = std::min(row_offset + num_rows, data->size()); + for (size_t i = row_offset; i < end; ++i) { + if (null_map != nullptr && (*null_map)[i] != 0) { + continue; + } + const float score = (*data)[i]; + // score is in (0, 1] whenever d is finite and non-negative, and within that + // range the radicand is provably non-negative: a real value above 1 cannot + // round down to below 1.0F in f32, so 1/score >= 1. The useful range + // therefore never reaches either guard below -- they exist because the + // distances come straight out of the closed-source lumina kernel with no + // clamp, and computing the inverse on an illegal score would produce a + // non-finite distance (score <= 0 -> 1/score is inf; score > 1 -> negative + // radicand -> NaN), which makes the coordinator's Top-N comparisons + // ill-defined. Both emit kCorruptScoreDistance instead: +inf sorts last in + // an ASC Top-N, so these rows can never displace a valid closer row. A + // score of exactly 0 is REACHABLE from a genuine huge distance — the exact + // fallback (vector-index.search-mode=full) computes the squared L2 in f32, + // and norms above ~1.8e19 overflow to +inf, so 1/(1+d^2) rounds to 0; + // publishing a sentinel ahead of finite rows would invert the final TopN. + // + // A score slightly above 1 is NOT corruption: the kernel computes squared + // L2 as ||a||^2 + ||b||^2 - 2, which can yield a small negative + // residual (catastrophic cancellation) when a ~= b -- a near-exact hit. + // Clamp those to distance 0; only scores past kScoreAboveOneTolerance are + // treated as genuinely corrupt. + if (UNLIKELY(score <= 0.0F)) { + LOG(WARNING) << "paimon-rust L2 score_to_distance: corrupt score " << score << " (row " + << i << ", expect score in (0, 1]), " + << "distance " << kCorruptScoreDistance; + (*data)[i] = kCorruptScoreDistance; + } else if (UNLIKELY(score > 1.0F + kScoreAboveOneTolerance)) { + LOG(WARNING) << "paimon-rust L2 score_to_distance: corrupt score " << score << " (row " + << i << ", expect score in (0, 1]), " + << "distance " << kCorruptScoreDistance; + (*data)[i] = kCorruptScoreDistance; + } else if (score > 1.0F) { + // Numerical artifact of a near-exact hit (see kScoreAboveOneTolerance): + // clamp to distance 0 instead of sentineling a legitimate best match. + (*data)[i] = 0.0F; + } else { + // score == 1.0F gives exactly 0 here -- a legal exact hit, which is why + // this must not share a branch with the negative-radicand case. + (*data)[i] = std::sqrt(1.0F / score - 1.0F); + } + } + // mutate() may have unshared a reference-counted column; put the (possibly + // new) mutable instance back into the block. + column_with_type.column = std::move(column); +} + void PaimonRustTableReader::_close_split_reader() { if (!_handles) { return; @@ -550,7 +977,7 @@ Status PaimonRustTableReader::_apply_predicate() { names.reserve(_projected_columns.size()); types.reserve(_projected_columns.size()); for (const auto& col : _projected_columns) { - if (col.is_partition_key) { + if (col.is_partition_key || col.name == kPaimonSearchDistanceColumn) { continue; } names.push_back(col.name); @@ -588,6 +1015,33 @@ Status PaimonRustTableReader::_fill_block_from_record_batch( DORIS_CHECK(block != nullptr); std::unordered_set materialized_indices; materialized_indices.reserve(_projected_columns.size()); + + // PK-vector (ANN) search mode: paimon-rust appends the score field + // (__paimon_search_score) to every batch, and the reader must convert it into + // the distance FE asked for (__paimon_search_score_to_dis) in place, after the + // fill loop. Locate the output position up front: FE only adds the distance + // column when it rewrote the query into a vector search (see + // PushDownVectorTopNIntoPaimonScan), so _score_transform being set implies the + // column is projected. Failing loudly here beats the alternative: the fill loop + // would silently skip the lookup, leaving the column empty while every other + // column holds `rows` -- an unattributable "sizes of columns don't match" + // downstream. + std::optional distance_output_idx; + size_t distance_row_offset = 0; + if (_score_transform.has_value()) { + auto it = _output_name_to_idx.find(kPaimonSearchDistanceColumn); + if (it == _output_name_to_idx.end()) { + return Status::InternalError( + "paimon-rust vector search: block has no '{}' column (FE/BE column name " + "contract broken)", + kPaimonSearchDistanceColumn); + } + distance_output_idx = it->second; + // Must be read BEFORE the fill loop: read_column_from_arrow appends, and the + // column may already hold converted rows from a previous batch of this split. + distance_row_offset = block->get_by_position(it->second).column->size(); + } + { auto columns_guard = block->mutate_columns_scoped(); auto& columns = columns_guard.mutable_columns(); @@ -599,10 +1053,13 @@ Status PaimonRustTableReader::_fill_block_from_record_batch( // Projected column names are FE-normalized to lowercase. // paimon-rust's case_sensitive=false setting also case-folds column // names in the schema output, so exact match works — but tolerate - // mixed-case Rust output by folding here as well. + // mixed-case Rust output by folding here as well. The score field is + // additionally renamed onto the distance column it feeds (see + // arrow_field_to_block_column). + const std::string block_name = arrow_field_to_block_column(field->name()); auto it = _output_name_to_idx.find(field->name()); if (it == _output_name_to_idx.end()) { - it = _output_name_to_idx.find(to_lower(field->name())); + it = _output_name_to_idx.find(block_name); } if (it == _output_name_to_idx.end()) { // Skip columns that are not in the block (e.g. columns dropped by @@ -610,6 +1067,30 @@ Status PaimonRustTableReader::_fill_block_from_record_batch( continue; } const auto output_idx = it->second; + if (distance_output_idx.has_value() && output_idx == *distance_output_idx) { + // Check the Arrow width before the serde reinterprets the buffer. For + // numerics read_column_from_arrow does NOT validate the source type: it + // casts buffers[1] to the *destination* CppType and inserts `rows` of + // them. The destination is pinned to FLOAT by FE + // (PaimonVectorSearch.createDistanceColumn), so + // - a wider field (Float64) yields `rows` garbage f32 values, each half + // of a double's bits. The column still grows by exactly `rows`, so + // the guard below passes and the query returns a wrongly-ordered + // Top-K with no error anywhere. + // - a narrower field (HALF_FLOAT) reads rows*4 bytes out of a + // rows*2 byte buffer, i.e. past its end. + // paimon-rust builds this field as Float32 unconditionally + // (materialize_search_result), so this only fires if that changes -- + // which is exactly when a loud failure beats either outcome above. + if (batch->column(c)->type_id() != arrow::Type::FLOAT) { + return Status::InternalError( + "paimon-rust vector search: arrow field '{}' has type {}, expected " + "float32 -- the block's '{}' column is FLOAT and the arrow buffer " + "is reinterpreted as f32 without conversion", + field->name(), batch->column(c)->type()->ToString(), + kPaimonSearchDistanceColumn); + } + } if (!materialized_indices.emplace(output_idx).second) { return Status::InternalError("paimon-rust returned duplicate column '{}'", field->name()); @@ -625,6 +1106,32 @@ Status PaimonRustTableReader::_fill_block_from_record_batch( } } } + + if (distance_output_idx.has_value()) { + // Guard the two assumptions _convert_score_to_distance depends on, both of + // which fail SILENTLY otherwise -- the block stays well-formed and the query + // returns a wrongly-ordered Top-K with no error anywhere: + // + // - distance_row_offset was captured BEFORE the fill loop. If a refactor + // moves that capture below the loop, the offset already includes this + // batch, so `end = min(offset + rows, size)` collapses to `offset` and + // the conversion loop runs zero times, leaving raw scores in the column. + // Since a score is higher-is-better, an l2 ORDER BY then returns the LEAST + // similar rows. + // - the Arrow batch actually carried the score field. If paimon-rust renames + // it, the fill loop's lookup misses and the column is never appended to. + const size_t size_after = block->get_by_position(*distance_output_idx).column->size(); + if (size_after != distance_row_offset + rows) { + return Status::InternalError( + "paimon-rust vector search: '{}' column grew from {} to {}, expected {} " + "(batch has {} rows) -- either the row offset was captured after the fill " + "loop, or the arrow batch is missing the '{}' field", + kPaimonSearchDistanceColumn, distance_row_offset, size_after, + distance_row_offset + rows, rows, kPaimonRustScoreField); + } + _convert_score_to_distance(block, static_cast(*distance_output_idx), + distance_row_offset, rows, *_score_transform); + } // Partition columns and other projected columns absent from the arrow batch // are back-filled from split metadata / defaults. RETURN_IF_ERROR(_fill_non_arrow_columns(block, rows, materialized_indices)); @@ -797,6 +1304,20 @@ std::optional PaimonRustTableReader::_resolve_branch( return std::nullopt; } +#ifdef BE_TEST +std::vector PaimonRustTableReader::TEST_build_read_columns( + const TFileRangeDesc& range) const { + // _build_read_columns judges vector mode off _current_range; borrow it for the + // test by reading the columns through the same member path. + TFileRangeDesc& current = const_cast(_current_range); + TFileRangeDesc saved = current; + current = range; + std::vector columns = _build_read_columns(); + current = saved; + return columns; +} +#endif + std::vector PaimonRustTableReader::_build_read_columns() const { std::vector columns; columns.reserve(_projected_columns.size()); @@ -804,6 +1325,15 @@ std::vector PaimonRustTableReader::_build_read_columns() const { if (column.is_partition_key) { continue; } + // The PK-vector search distance is reader-produced and auto-appended by + // paimon-rust under its own field name; it is never a projectable table + // column, so exclude it from the projection — but only in vector mode: + // __paimon_search_score_to_dis is also a legal Paimon user column name, and + // skipping it on an ordinary rust scan would drop the real column and + // fill NULLs through _fill_non_arrow_columns. + if (_is_vector_mode(_current_range) && column.name == kPaimonSearchDistanceColumn) { + continue; + } columns.emplace_back(column.name); } return columns; diff --git a/be/src/format_v2/table/paimon_rust_table_reader.h b/be/src/format_v2/table/paimon_rust_table_reader.h index 0c04f2f1f04d59..c4d4a2e94e9c40 100644 --- a/be/src/format_v2/table/paimon_rust_table_reader.h +++ b/be/src/format_v2/table/paimon_rust_table_reader.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include #include #include #include @@ -30,6 +32,7 @@ #include "cctz/time_zone.h" #include "common/status.h" #include "format_v2/table_reader.h" +#include "gen_cpp/PlanNodes_types.h" // TVectorMetric #include "runtime/runtime_profile.h" namespace arrow { @@ -61,6 +64,75 @@ class PaimonRustTableReader final : public format::TableReader { PaimonRustTableReader(); ~PaimonRustTableReader() override; + // How paimon-rust encoded the distance into its __paimon_search_score field, and + // hence which transform the reader has to undo before the value lands in the + // __paimon_search_score_to_dis block column. One enumerator per metric Doris can + // push down (v1: l2 / inner_product; cosine is rejected by the FE rule and by + // parse_score_transform, so it deliberately has no enumerator here). + // + // Adding a metric is meant to be compiler-guided: add the enumerator, and the + // switch in _convert_score_to_distance stops compiling (-Wswitch is an error in + // be/src) until its inverse is written. Do not add a `default:` to that switch + // or that guarantee is lost. + enum class ScoreTransform { + // score = 1/(1 + squaredL2). Inverse: sqrt(1/score - 1). + L2, + // score = the inner product itself. Inverse: identity. + InnerProduct, + }; + + // Map an Arrow field name onto the block column name that receives its values. + // + // paimon-rust resolves projections case-insensitively, so the Arrow batch carries + // the table's canonical field casing (e.g. `is_QT`) while block columns are + // FE-normalized to lowercase (`is_qt`). Fold the Rust-side name down so the two match. + // + // The vector-search field is additionally renamed: paimon-rust calls it + // kPaimonRustScoreField ("__paimon_search_score") and puts a score in it, but the + // reader converts that to a distance in place, so FE asks for the column as + // kPaimonSearchDistanceColumn ("__paimon_search_score_to_dis"). The fill loop + // skips Arrow fields it cannot match (they may legitimately be absent from the + // block), so without this alias the rename would not fail loudly -- the distance + // column would just come back empty. + static std::string arrow_field_to_block_column(const std::string& arrow_name); + + // Map a thrift TVectorMetric onto its score transform. Unknown or unsupported + // metrics (including COSINE) are an error rather than a silent fallback: guessing + // would return wrong distances instead of failing the query. Takes the thrift enum + // directly so there is no string round-trip between FE and the ScoreTransform enum. + static Status parse_score_transform(TVectorMetric::type metric, ScoreTransform* transform); + + // Rewrite the reader-produced __paimon_search_score_to_dis column in place, + // turning paimon-rust's score back into the distance the Doris function is + // defined to return (see ScoreTransform for the per-metric formulas). + // + // Only rewrites [row_offset, row_offset + num_rows): block columns are + // append-only and a block is reused across get_block calls, so starting at 0 + // would square-root earlier rows again. Depends on nothing but its arguments, + // hence static and public (also what makes it unit-testable without the + // paimon-rust FFI handles). + // + // A score outside l2's legal (0, 1] range is index corruption, and computing + // the inverse anyway would yield a non-finite distance (see the two cases + // below). Such rows are emitted as +inf instead: +inf sorts LAST in an ASC + // Top-K, so a corrupt or infinite row can never displace a valid closer row + // from the coordinator's final ordering. A score of exactly 0 is reachable + // from a genuine (huge) distance — the exact fallback in + // vector-index.search-mode=full computes the squared L2 in f32, so norms + // above ~1.8e19 overflow to +inf and score = 1/(1+d^2) rounds to 0 — which + // is why the boundary maps to +inf rather than to an error. + static constexpr float kCorruptScoreDistance = std::numeric_limits::infinity(); + // paimon-rust's lumina kernel computes the squared L2 distance as + // ||a||^2 + ||b||^2 - 2 (vectorized dot), which suffers catastrophic + // cancellation when a ~= b: the three terms nearly cancel and the residual + // can be a small NEGATIVE number, so score = 1/(1+d^2) lands slightly ABOVE 1 + // (e.g. 1.00294). That is a numerical artifact for an ~exact hit, not index + // corruption -- clamp it to distance 0. Only a score clearly past 1 (beyond + // this tolerance) is treated as a genuine corrupt row and sentineled. + static constexpr float kScoreAboveOneTolerance = 1e-2F; + static void _convert_score_to_distance(Block* block, uint32_t block_idx, size_t row_offset, + size_t num_rows, ScoreTransform transform); + Status init(format::TableReadOptions&& options) override; // Each split's Rust read builder applies the batch size stored by the base class // before opening the Arrow reader. @@ -85,6 +157,9 @@ class PaimonRustTableReader final : public format::TableReader { void TEST_set_projected_columns(std::vector columns) { _projected_columns = std::move(columns); } + // The projection the reader would request for the given range; exposed so the + // synthetic-distance-column exclusion (vector mode only) can be unit-tested. + std::vector TEST_build_read_columns(const TFileRangeDesc& range) const; void TEST_set_partition_values(std::map values) { _partition_values = std::move(values); } @@ -108,9 +183,31 @@ class PaimonRustTableReader final : public format::TableReader { // Fail fast (without filesystem IO) when the schema-json pipeline fields are // missing from the split; the error text points at an FE/BE protocol mismatch. Status _validate_rust_split(const TFileRangeDesc& range) const; + // Validate the external_search_request thrift fields before any use (mirrors + // LanceTableReader::_validate_external_search_request): one pass over __isset + // and value-range checks so the vector build path can use the fields unchecked. + Status _validate_external_search_request() const; + // True when this split carries a PK-vector search payload and the reader must + // open the vector-search read path instead of the normal split read. + static bool _is_vector_mode(const TFileRangeDesc& range); + // Resolve the table root path + schema json and open (or reuse the cached) + // paimon_table handle from them. Shared by the normal-read and vector-search + // build paths; fills _handles->table and _opened_table_key. + Status _open_paimon_table(const TFileRangeDesc& range); // Open (or reuse the cached) paimon_table and build the per-split read // pipeline: read_builder -> projection -> filter -> plan -> arrow reader. Status _open_split_reader(const TFileRangeDesc& range); + // Open the table and build a primary-key vector (ANN) search reader over the + // FE-planned bucket split. Used when the scan range carries a + // TPaimonVectorPayload: the returned Arrow stream is best-first and carries the + // reader-produced distance column (__paimon_search_score_to_dis, converted + // from paimon-rust's __paimon_search_score field). + Status _open_vector_split_reader(const TFileRangeDesc& range); + // Fill the output block from one arrow record batch: arrow columns are + // matched by name (exact then lower-case, v1 semantics) onto fixed projected + // positions, then _fill_non_arrow_columns back-fills the rest. + Status _fill_block_from_record_batch(const std::shared_ptr& batch, + Block* block, size_t rows); // Drop the per-split pipeline (read_builder .. record batch reader), keeping // the cached table handle for the next split of the same table. void _close_split_reader(); @@ -120,11 +217,6 @@ class PaimonRustTableReader final : public format::TableReader { // apply it to the read builder. Best effort: non-convertible conjuncts are // dropped silently and re-applied by the scanner row-level filter. Status _apply_predicate(); - // Fill the output block from one arrow record batch: arrow columns are - // matched by name (exact then lower-case, v1 semantics) onto fixed projected - // positions, then _fill_non_arrow_columns back-fills the rest. - Status _fill_block_from_record_batch(const std::shared_ptr& batch, - Block* block, size_t rows); // Materialize projected columns the arrow batch did not contain: partition // constants, default expressions, or a default/NULL fill as a last resort. Status _fill_non_arrow_columns(Block* block, size_t rows, @@ -169,6 +261,12 @@ class PaimonRustTableReader final : public format::TableReader { std::unordered_map _output_name_to_idx; TFileRangeDesc _current_range; cctz::time_zone _ctz; + // Which score transform the incoming __paimon_search_score field needs undone, + // derived from the FE-supplied metric when the vector reader is built. Empty + // outside vector mode (and only then): an unset or unknown metric fails the + // scan rather than defaulting, since guessing l2 would wrongly sqrt + // inner_product results. + std::optional _score_transform; // EOF belongs to the current split; reset when a new split is prepared. bool _split_eof = false; diff --git a/be/test/format_v2/table/paimon_rust_table_reader_test.cpp b/be/test/format_v2/table/paimon_rust_table_reader_test.cpp index 72094f3e020a69..c909e219758bf9 100644 --- a/be/test/format_v2/table/paimon_rust_table_reader_test.cpp +++ b/be/test/format_v2/table/paimon_rust_table_reader_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include "cctz/time_zone.h" +#include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_const.h" #include "core/column/column_nullable.h" @@ -39,6 +41,7 @@ #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" #include "format_v2/column_data.h" +#include "format_v2/table/paimon_rust_predicate_converter.h" #include "gen_cpp/PlanNodes_types.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" @@ -235,6 +238,272 @@ TEST_F(PaimonRustTableReaderTest, AdaptiveBatchSizeBoundsWideStringReads) { ASSERT_TRUE(reader.close().ok()); } +TEST_F(PaimonRustTableReaderTest, ValidatesVectorSplit) { + PaimonRustTableReader reader; + + auto make_vector_range = [] { + // Vector mode: paimon_split is deliberately an empty string (FE avoids + // serializing the same DataSplit twice) and the payload is the serialized + // BucketVectorSearchSplit. + auto range = make_rust_range(); + range.table_format_params.paimon_params.__set_paimon_split(""); + TPaimonVectorPayload payload; + payload.__set_payload_type(TPaimonVectorPayloadType::BUCKET_SPLIT_BYTES); + payload.__set_bytes("PKVSPLIT-dummy"); + range.table_format_params.paimon_params.__set_vector_payload(payload); + return range; + }; + + // Missing vector_payload: a protocol error, not a fall-through to the empty + // paimon_split check (whose error text would point the reader at the wrong field). + auto range = make_vector_range(); + range.table_format_params.paimon_params.__isset.vector_payload = false; + auto status = reader.TEST_validate_rust_split(range); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("missing vector_payload"), std::string::npos) << status; + + // Empty payload bytes are equally a protocol error. + range = make_vector_range(); + range.table_format_params.paimon_params.vector_payload.bytes.clear(); + status = reader.TEST_validate_rust_split(range); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("missing vector_payload"), std::string::npos) << status; + + // A complete vector range validates cleanly -- including the empty paimon_split, + // which the normal path would reject. + EXPECT_TRUE(reader.TEST_validate_rust_split(make_vector_range()).ok()); +} + +TEST_F(PaimonRustTableReaderTest, OrdinaryScanKeepsSyntheticDistanceNamedColumn) { + // __paimon_search_score_to_dis is also a legal Paimon user column name. The + // synthetic-distance exclusion must therefore apply ONLY to a real vector + // split: an ordinary rust scan of `SELECT id, __paimon_search_score_to_dis` + // over a table that really has such a column must keep reading it (skipping + // it would drop the column from the read projection and fill NULLs through + // _fill_non_arrow_columns). + PaimonRustTableReader reader; + reader.TEST_set_projected_columns( + {make_column("id", std::make_shared()), + make_column(kPaimonSearchDistanceColumn, std::make_shared())}); + + // Ordinary split: the same-named real column rides in the projection. + TFileRangeDesc ordinary = make_rust_range(); + auto ordinaryColumns = reader.TEST_build_read_columns(ordinary); + ASSERT_EQ(2u, ordinaryColumns.size()); + EXPECT_EQ("id", ordinaryColumns[0]); + EXPECT_EQ(kPaimonSearchDistanceColumn, ordinaryColumns[1]); + + // Vector split (empty paimon_split + vector_payload): the synthetic slot is + // reader-produced by paimon-rust itself, so it must NOT be requested. + TFileRangeDesc vectorRange = make_rust_range(); + vectorRange.table_format_params.paimon_params.__set_paimon_split(""); + TPaimonVectorPayload payload; + payload.__set_payload_type(TPaimonVectorPayloadType::BUCKET_SPLIT_BYTES); + payload.__set_bytes("PKVSPLIT-dummy"); + vectorRange.table_format_params.paimon_params.__set_vector_payload(payload); + auto vectorColumns = reader.TEST_build_read_columns(vectorRange); + ASSERT_EQ(1u, vectorColumns.size()); + EXPECT_EQ("id", vectorColumns[0]); +} + +// ---------------------------------------------------------------- score -> distance +// +// paimon-rust encodes the distance as a score per metric (see +// convert_distance_to_score in crates/paimon/src/lumina/reader.rs); the reader has +// to undo exactly that so the column means what the Doris function returns. For l2 +// the score is 1/(1 + SQUARED distance), hence sqrt(1/score - 1). The reference +// points below are exact in binary floating point. + +using ScoreTransform = PaimonRustTableReader::ScoreTransform; +static constexpr auto L2 = ScoreTransform::L2; +static constexpr auto INNER_PRODUCT = ScoreTransform::InnerProduct; + +// A one-column block holding `scores` in a (possibly nullable) FLOAT column, +// standing in for what the arrow fill loop leaves in the distance column. +static Block _score_block(const std::vector& scores, bool nullable) { + DataTypePtr type = std::make_shared(); + if (nullable) { + type = std::make_shared(type); + } + MutableColumnPtr column = type->create_column(); + for (float score : scores) { + column->insert_data(reinterpret_cast(&score), sizeof(float)); + } + Block block; + block.insert(ColumnWithTypeAndName(std::move(column), type, kPaimonSearchDistanceColumn)); + return block; +} + +// The float at `row` of the block's only column, whether nullable or not. +static float _score_at(const Block& block, size_t row) { + const auto& column = block.get_by_position(0).column; + if (const auto* nullable = check_and_get_column(column.get())) { + return assert_cast(nullable->get_nested_column()).get_data()[row]; + } + return assert_cast(*column).get_data()[row]; +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceL2) { + // score 1.0 -> squared 0 -> distance 0 (an exact hit) + // score 0.5 -> squared 1 -> distance 1 + // score 0.2 -> squared 4 -> distance 2 + // + // Keep the score == 1.0 row: it is the legal boundary, sitting immediately next + // to the score > 1 corruption case below, and 1/1 - 1 is exactly 0 in floating + // point. A guard phrased as "sqrt(radicand) if radicand > 0, else sentinel" + // passes every other case here while misreporting exact hits as corrupt. + Block block = _score_block({1.0F, 0.5F, 0.2F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 3, L2); + + EXPECT_FLOAT_EQ(0.0F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(1.0F, _score_at(block, 1)); + EXPECT_FLOAT_EQ(2.0F, _score_at(block, 2)); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceClampsCatastrophicCancellation) { + // The lumina kernel computes squared L2 as ||a||^2 + ||b||^2 - 2, which can + // yield a small negative residual when a ~= b, i.e. score slightly ABOVE 1. That + // is a near-exact hit, not corruption: it clamps to distance 0. Only a score past + // kScoreAboveOneTolerance maps to kCorruptScoreDistance. + Block block = _score_block({1.0F + 1e-4F, 1.0F + 5e-2F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 2, L2); + + EXPECT_FLOAT_EQ(0.0F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreDistance, _score_at(block, 1)); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceSkipsRowsBeforeOffset) { + // Block columns are append-only and a block is reused across get_block calls + // when a batch is fully filtered out, so rows converted by an earlier call must + // be left alone. Here rows 0-1 already hold distances; only the two raw scores + // appended after them may be touched. + Block block = _score_block({0.0F, 2.0F, 0.5F, 0.2F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, /*row_offset=*/2, + /*num_rows=*/2, L2); + + // Untouched: converting again would turn distance 2.0 into sqrt(1/2 - 1) garbage. + EXPECT_FLOAT_EQ(0.0F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(2.0F, _score_at(block, 1)); + // Converted. + EXPECT_FLOAT_EQ(1.0F, _score_at(block, 2)); + EXPECT_FLOAT_EQ(2.0F, _score_at(block, 3)); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceNullable) { + // FE declares the score column non-nullable, but the slot decides the actual + // block column, so the nullable shape must work too: null rows stay null and + // their payload is not read. + Block block = _score_block({0.5F, 0.0F, 0.2F}, /*nullable=*/true); + auto nullable = IColumn::mutate(std::move(block.get_by_position(0).column)); + assert_cast(nullable.get())->get_null_map_data()[1] = 1; + block.get_by_position(0).column = std::move(nullable); + + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 3, L2); + + EXPECT_FLOAT_EQ(1.0F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(2.0F, _score_at(block, 2)); + const auto* null_column = + check_and_get_column(block.get_by_position(0).column.get()); + ASSERT_NE(null_column, nullptr); + EXPECT_EQ(1, null_column->get_null_map_data()[1]); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceFlagsOutOfRangeScores) { + // A well-formed index keeps d >= 0 and score in (0, 1], but the distances + // come out of the closed-source lumina kernel unclamped, and a score of + // exactly 0 is also REACHABLE from a genuine huge distance: the exact + // fallback (vector-index.search-mode=full) computes squared L2 in f32, so + // norms above ~1.8e19 overflow to +inf and 1/(1+d^2) rounds to 0. Every + // out-of-range case maps to kCorruptScoreDistance (+inf): it sorts LAST in + // an ASC Top-N, so it can never displace a valid closer row, and it cannot + // invert the coordinator's final ordering the way a negative sentinel + // (row 0 beating real finite distances) did. + Block block = _score_block({0.0F, -1.0F, 1.5F, 0.5F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 4, L2); + + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreDistance, _score_at(block, 0)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreDistance, _score_at(block, 1)); + EXPECT_FLOAT_EQ(PaimonRustTableReader::kCorruptScoreDistance, _score_at(block, 2)); + // A legal score between the corrupt rows still converts exactly. + EXPECT_FLOAT_EQ(1.0F, _score_at(block, 3)); + for (size_t i = 0; i < 3; ++i) { + EXPECT_TRUE(std::isinf(_score_at(block, i))) << "row " << i; + } +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceEmptyBatch) { + // An empty batch must be a no-op rather than an out-of-range write. + Block block = _score_block({0.5F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, /*row_offset=*/1, + /*num_rows=*/0, L2); + EXPECT_FLOAT_EQ(0.5F, _score_at(block, 0)); +} + +TEST_F(PaimonRustTableReaderTest, ConvertScoreToDistanceInnerProductIsIdentity) { + // For inner_product the score IS the value the Doris function returns, so every + // row must survive untouched -- including negative and out-of-(0,1] values, which + // are legal here (an inner product is unbounded) and must NOT be mistaken for + // the out-of-range l2 scores that get a corruption sentinel. + Block block = _score_block({0.5F, 0.0F, -3.25F, 42.0F}, /*nullable=*/false); + PaimonRustTableReader::_convert_score_to_distance(&block, 0, 0, 4, INNER_PRODUCT); + + EXPECT_FLOAT_EQ(0.5F, _score_at(block, 0)); + EXPECT_FLOAT_EQ(0.0F, _score_at(block, 1)); + EXPECT_FLOAT_EQ(-3.25F, _score_at(block, 2)); + EXPECT_FLOAT_EQ(42.0F, _score_at(block, 3)); +} + +// ---------------------------------------------------------------- metric parsing + +TEST_F(PaimonRustTableReaderTest, ParseScoreTransformAcceptsSupportedMetrics) { + ScoreTransform transform = INNER_PRODUCT; + ASSERT_TRUE(PaimonRustTableReader::parse_score_transform(TVectorMetric::L2, &transform).ok()); + EXPECT_EQ(L2, transform); + + ASSERT_TRUE(PaimonRustTableReader::parse_score_transform(TVectorMetric::DOT_PRODUCT, &transform) + .ok()); + EXPECT_EQ(INNER_PRODUCT, transform); +} + +TEST_F(PaimonRustTableReaderTest, ParseScoreTransformRejectsUnsupportedMetrics) { + // Must fail rather than fall back to l2: guessing would sqrt an inner product, + // or return a cosine score where a distance was promised. v1 does not push + // cosine down, so seeing it here means FE and BE disagree. + ScoreTransform transform = L2; + for (auto metric : {TVectorMetric::COSINE, TVectorMetric::DEFAULT, TVectorMetric::HAMMING}) { + auto status = PaimonRustTableReader::parse_score_transform(metric, &transform); + EXPECT_FALSE(status.ok()) << "metric " << static_cast(metric) << " must be rejected"; + EXPECT_NE(status.to_string().find("unsupported distance metric"), std::string::npos) + << status; + } +} + +// ------------------------------------------------- arrow field -> block column + +TEST_F(PaimonRustTableReaderTest, ArrowFieldToBlockColumnRenamesScoreField) { + // FE names the output column "..._to_dis" because the reader turns the score + // into a distance, but paimon-rust keeps emitting the Arrow field as + // "__paimon_search_score". The fill loop silently skips Arrow fields it cannot + // match, so if this mapping regressed the distance column would come back + // empty rather than error out -- hence the explicit assertion. + EXPECT_EQ(kPaimonSearchDistanceColumn, + PaimonRustTableReader::arrow_field_to_block_column(kPaimonRustScoreField)); + // paimon-rust may return the canonical (any-case) field name. + EXPECT_EQ(kPaimonSearchDistanceColumn, + PaimonRustTableReader::arrow_field_to_block_column("__Paimon_Search_Score")); +} + +TEST_F(PaimonRustTableReaderTest, ArrowFieldToBlockColumnLowercasesOtherFields) { + // Ordinary table columns are only case-folded: paimon-rust resolves projections + // case-insensitively and returns the table's canonical casing, while block + // columns are FE-normalized to lowercase. + EXPECT_EQ("is_qt", PaimonRustTableReader::arrow_field_to_block_column("is_QT")); + EXPECT_EQ("id", PaimonRustTableReader::arrow_field_to_block_column("id")); + // Not the score field, so it must NOT be rewritten to the distance column. + EXPECT_EQ("__paimon_search_score_extra", + PaimonRustTableReader::arrow_field_to_block_column("__paimon_search_score_extra")); +} + TEST_F(PaimonRustTableReaderTest, TableLevelCountEmitsSyntheticRows) { // COUNT(*) with a table-level row count takes the base-class metadata path: // prepare_split never opens the rust pipeline and get_block emits synthetic rows. diff --git a/fe/be-java-extensions/paimon-connector/pom.xml b/fe/be-java-extensions/paimon-connector/pom.xml index afbc5a77dece7d..6460b9aa0f380d 100644 --- a/fe/be-java-extensions/paimon-connector/pom.xml +++ b/fe/be-java-extensions/paimon-connector/pom.xml @@ -92,6 +92,17 @@ under the License. paimon-connector ${project.basedir}/target/ + + org.apache.maven.plugins + maven-surefire-plugin + + + + --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index f8efad58e80995..e884e42bd1b54d 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -31,6 +31,7 @@ import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VectorType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -239,12 +240,30 @@ private Variant getVariant() { @Override public void unpackArray(List values) { - InternalArray recordArray = record.getArray(idx); + // A Paimon VECTOR is exposed to Doris as ARRAY (PaimonUtil + // .paimonPrimitiveTypeToDorisType), so it lands here too -- but it is a distinct + // type, not an ArrayType subclass, and it has its own physical layout: + // readVectorData carries a 4-byte element count that readArrayData does not. + // Hence both the accessor and the element type must be chosen by type: + // - getVector vs getArray: BinaryRow.getArray on a vector field would decode + // vector bytes as a BinaryArray, i.e. silently wrong values rather than an + // error. (The columnar path special-cases the vector column vector, which + // is why only this cast crashed.) + // - the element type cast: this is where "VectorType cannot be cast to + // ArrayType" came from when reading a vector column through the JNI scanner. + InternalArray recordArray; + DataType elementPaimonType; + if (dataType instanceof VectorType) { + recordArray = record.getVector(idx); + elementPaimonType = ((VectorType) dataType).getElementType(); + } else { + recordArray = record.getArray(idx); + elementPaimonType = ((ArrayType) dataType).getElementType(); + } if (arrayValues == null) { arrayValues = new ArrayList<>(); } ColumnType elementDorisType = dorisType.getChildTypes().get(0); - DataType elementPaimonType = ((ArrayType) dataType).getElementType(); for (int i = 0; i < recordArray.size(); i++) { values.add(reuseColumnValue(arrayValues, i, (DataGetters) recordArray, i, elementDorisType, elementPaimonType)); diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java index 6304bfdabbe662..f474a1c70db0dd 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java @@ -24,6 +24,7 @@ import org.apache.paimon.types.DataType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VectorType; import java.util.ArrayList; import java.util.List; @@ -81,6 +82,16 @@ private static DataType projectRow(DataType tableType, ColumnType requiredType) } private static DataType projectArray(DataType tableType, ColumnType requiredType) { + // Doris maps a Paimon VECTOR column to ARRAY (PaimonUtil.paimonPrimitiveTypeToDorisType), + // so a VECTOR column arrives here with an ARRAY request. The Doris ARRAY is only the + // logical view: the physical read must stay the VECTOR type, whose binary layout + // differs from an array's (readVectorData carries a 4-byte element count that + // readArrayData does not -- see PaimonColumnValue.unpackArray, which dispatches on + // this type being VectorType). Forcing an ArrayType read would make the reader + // decode vector bytes as a BinaryArray, i.e. silently wrong values. + if (tableType instanceof VectorType) { + return tableType; + } if (!(tableType instanceof ArrayType)) { throw incompatibleType(requiredType, tableType); } diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java index be457571f0be87..d2d944b078eef8 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java @@ -44,6 +44,7 @@ import org.apache.paimon.types.VarBinaryType; import org.apache.paimon.types.VarCharType; import org.apache.paimon.types.VariantType; +import org.apache.paimon.types.VectorType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,6 +183,22 @@ public PaimonColumnType visit(ArrayType arrayType) { return paimonColumnType; } + @Override + public PaimonColumnType visit(VectorType vectorType) { + // Same shape as visit(ArrayType): a Paimon VECTOR is exposed to + // Doris as ARRAY, matching PaimonUtil.paimonPrimitiveTypeToDorisType on + // the FE side. DataTypeVisitor declares visit(VectorType) separately from + // visit(ArrayType), so without this override a vector column would fall through + // to defaultMethod() -> Type.UNSUPPORTED, which only logs. + // + // The declared length is dropped because Doris ARRAY has no fixed-size variant; + // Paimon enforces the dimension on write. + PaimonColumnType paimonColumnType = new PaimonColumnType(Type.ARRAY); + ColumnType elementColumnType = fromPaimonType("dummy-element", vectorType.getElementType()); + paimonColumnType.setChildTypes(Collections.singletonList(elementColumnType)); + return paimonColumnType; + } + @Override public PaimonColumnType visit(MultisetType multisetType) { return this.defaultMethod(multisetType); diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java index bce0c5950e7863..2f95a11ce1eb03 100644 --- a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java @@ -28,15 +28,12 @@ import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.Property; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.BufferFileReader; import org.apache.paimon.disk.BufferFileWriter; import org.apache.paimon.disk.FileIOChannel; import org.apache.paimon.disk.IOManager; import org.apache.paimon.disk.IOManagerImpl; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.DelegatedFileStoreTable; @@ -341,21 +338,16 @@ public void testFallbackWrapperOrderMatchesPaimonFactoryPrecedence() { } @Test - public void testBackendCapTraversesPrivilegeDelegate() { + public void testBackendCapTraversesPlainDelegate() { FileStoreTable main = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1")); FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128")); - PrivilegeChecker checker = (PrivilegeChecker) Proxy.newProxyInstance( - PrivilegeChecker.class.getClassLoader(), - new Class[] {PrivilegeChecker.class}, - (proxy, method, args) -> null); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(main, fallback, true), checker, - Identifier.create("db", "table")); + FileStoreTable delegate = new PlainDelegateFileStoreTable( + new FallbackReadFileStoreTable(main, fallback, true)); Table safe = PaimonJniScanner.applyBackendManifestParallelism( - privileged, null, 64); + delegate, null, 64); FileStoreTable planningTable = safe instanceof DelegatedFileStoreTable && !(safe instanceof FallbackReadFileStoreTable) ? ((DelegatedFileStoreTable) safe).wrapped() : (FileStoreTable) safe; @@ -390,19 +382,14 @@ public void testOldFeSystemWrapperPreservesIndependentFallbackLimits() throws Ex } @Test - public void testSystemWrapperExposesSafeFallbackBehindPrivilegeDelegate() throws Exception { + public void testSystemWrapperExposesSafeFallbackBehindPlainDelegate() throws Exception { FileStoreTable main = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1")); FileStoreTable fallback = serializableFileStoreTable(Collections.singletonMap( CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "2")); - PrivilegeChecker checker = (PrivilegeChecker) Proxy.newProxyInstance( - PrivilegeChecker.class.getClassLoader(), - new Class[] {PrivilegeChecker.class}, - (proxy, method, args) -> null); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(main, fallback, true), checker, - Identifier.create("db", "table")); - Table wrapper = SystemTableLoader.load("partitions", privileged); + FileStoreTable delegate = new PlainDelegateFileStoreTable( + new FallbackReadFileStoreTable(main, fallback, true)); + Table wrapper = SystemTableLoader.load("partitions", delegate); Table safe = PaimonJniScanner.applyBackendManifestParallelism( wrapper, null, 64); @@ -959,6 +946,10 @@ public Object invoke(Object proxy, Method method, Object[] args) { if ("rowType".equals(method.getName())) { return new RowType(Collections.emptyList()); } + if ("coreOptions".equals(method.getName())) { + // Paimon 2.1 system-table loading consults queryAuthEnabled() on the data table. + return new CoreOptions(options); + } if ("schema".equals(method.getName())) { return new TableSchema(0L, Collections.emptyList(), 0, Collections.emptyList(), Collections.emptyList(), options, ""); diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PlainDelegateFileStoreTable.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PlainDelegateFileStoreTable.java new file mode 100644 index 00000000000000..e57cdddbfdb26b --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PlainDelegateFileStoreTable.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.DelegatedFileStoreTable; +import org.apache.paimon.table.FileStoreTable; + +import java.util.Map; + +/** + * Minimal non-fallback delegate used to exercise the delegate-peeling paths that the removed + * Paimon privilege wrapper used to cover. + */ +final class PlainDelegateFileStoreTable extends DelegatedFileStoreTable { + + PlainDelegateFileStoreTable(FileStoreTable wrapped) { + super(wrapped); + } + + @Override + public FileStoreTable copy(Map dynamicOptions) { + return new PlainDelegateFileStoreTable(wrapped().copy(dynamicOptions)); + } + + @Override + public FileStoreTable copy(TableSchema newTableSchema) { + return new PlainDelegateFileStoreTable(wrapped().copy(newTableSchema)); + } + + @Override + public FileStoreTable copyWithoutTimeTravel(Map dynamicOptions) { + return new PlainDelegateFileStoreTable(wrapped().copyWithoutTimeTravel(dynamicOptions)); + } + + @Override + public FileStoreTable copyWithLatestSchema() { + return new PlainDelegateFileStoreTable(wrapped().copyWithLatestSchema()); + } + + @Override + public FileStoreTable switchToBranch(String branchName) { + return new PlainDelegateFileStoreTable(wrapped().switchToBranch(branchName)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 978ea0bd303be6..336fdf3266dde7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -42,6 +42,7 @@ import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccTable; import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.datasource.paimon.source.PaimonVectorSearch; import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; @@ -379,6 +380,17 @@ private void setColumnPositionMapping() if (colName.startsWith(Column.GLOBAL_ROWID_COL)) { continue; } + // Reader-produced Paimon PK-vector search distance: a synthetic output column + // filled by the BE reader by name, not present in the table's file schema, + // so it has no Parquet/ORC column position to map. Only skip it when the + // schema really does not carry the name: this method serves every external + // catalog, and a REAL column sharing the name (a Hive text table can + // legally define one) must keep its normal position mapping — skipping it + // would emit fewer column_idxs than the file has and fail the BE reader. + if (PaimonVectorSearch.SEARCH_DISTANCE_COLUMN.equalsIgnoreCase(colName) + && columnNames.stream().noneMatch(n -> n.equalsIgnoreCase(colName))) { + continue; + } Integer idx = columnNameMap.get(colName); if (idx == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 7996b06106d8df..4dcdf9d0275e91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -55,7 +55,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; import org.apache.paimon.partition.Partition; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.DataTable; @@ -301,12 +300,6 @@ public long fetchRowCount() { // the complete snapshot selected by TimeTravelUtil. return UNKNOWN_ROW_COUNT; } - if (table instanceof PrivilegedFileStoreTable) { - // Preserve SELECT authorization without planning. TimeTravelUtil calls tagManager(), - // which would incorrectly require INSERT permission on the privilege wrapper. - table.newScan(); - table = PaimonTableDecorators.unwrapToFallbackOrBase(table); - } if (options.queryAuthEnabled()) { table.catalogEnvironment().tableQueryAuth(options).auth(null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java index 5a422f6af5a9f2..63fb7c002e2c7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java @@ -22,7 +22,6 @@ import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.table.DelegatedFileStoreTable; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; @@ -193,20 +192,11 @@ private static Table normalizeManifestParallelism( } if (table instanceof DelegatedFileStoreTable) { - if (!(table instanceof PrivilegedFileStoreTable)) { - throw new IllegalArgumentException("Unsupported Paimon planning table delegate: " - + table.getClass().getName()); - } - FileStoreTable wrapped = ((DelegatedFileStoreTable) table).wrapped(); - FileStoreTable normalized = normalizeManifestParallelism( - wrapped, safeBound, materializeAbsent); - if (normalized == wrapped) { - return table; - } - // A delegate copy broadcasts one value to every fallback branch. Check authorization - // before peeling the privilege-only layer so branch-local limits remain independent. - ((FileStoreTable) table).newScan(); - return normalized; + // Paimon 2.1 dropped the privilege wrapper, so every delegate the factory produces is + // a FallbackReadFileStoreTable and was already rebuilt above. Any other delegate's + // copy() broadcasts one value through its private fallback tree, so reject it. + throw new IllegalArgumentException("Unsupported Paimon planning table delegate: " + + table.getClass().getName()); } if (!(table instanceof FileStoreTable)) { @@ -234,7 +224,7 @@ private static Table normalizeManifestParallelism( static boolean isWrappedFirst(FallbackReadFileStoreTable table) { Map options = table.options(); - // Keep this in the exact order used by Paimon 1.4.2 FileStoreTableFactory. The same + // Keep this in the exact order used by Paimon's FileStoreTableFactory. The same // wrapper represents chain, fallback and primary reads, but the SDK exposes no accessor // for its private wrappedFirst flag. Remove this inference when Paimon exposes one. if (ChainTableUtils.isChainTable(options)) { @@ -262,8 +252,8 @@ public static void validateEffectiveTable(Table table) { validateEffectiveTable(((FallbackReadFileStoreTable) table).other()); } if (table instanceof DelegatedFileStoreTable) { - // Privilege and other supported delegates can hide a fallback planner behind their - // own visible main options, so traverse the complete planning-handle chain. + // Delegates can hide a fallback planner behind their own visible main options, + // so traverse the complete planning-handle chain. validateEffectiveTable(((DelegatedFileStoreTable) table).wrapped()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableDecorators.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableDecorators.java index 7060b6ae4c36ec..a45c8577f24804 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableDecorators.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableDecorators.java @@ -30,8 +30,8 @@ private PaimonTableDecorators() { public static FileStoreTable unwrapToFallbackOrBase(FileStoreTable table) { FileStoreTable current = table; // Fallback dispatch requires the pair to be the system wrapper's immediate child. Paimon - // does not privilege-wrap system tables, so peeling privilege-only delegates matches its - // catalog semantics while retaining the one decorator that changes read routing. + // 2.1 ships no privilege delegates, so peeling any unknown delegate matches its catalog + // semantics while retaining the one decorator that changes read routing. while (current instanceof DelegatedFileStoreTable && !(current instanceof FallbackReadFileStoreTable)) { current = ((DelegatedFileStoreTable) current).wrapped(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 2a362ebebfdec4..fee62ba244cc55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -80,6 +80,7 @@ import org.apache.paimon.types.TimestampType; import org.apache.paimon.types.VarBinaryType; import org.apache.paimon.types.VarCharType; +import org.apache.paimon.types.VectorType; import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.InternalRowPartitionComputer; @@ -372,6 +373,27 @@ private static Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataT Type innerType = paimonPrimitiveTypeToDorisType(arrayType.getElementType(), enableVarbinaryMapping, enableTimestampTzMapping); return org.apache.doris.catalog.ArrayType.create(innerType, true); + case VECTOR: + // Paimon's fixed-size dense vector (VECTOR) maps to a plain + // ARRAY, same as case ARRAY above. That is what the rest of the stack already + // expects: l2_distance_approximate / inner_product_approximate are declared + // over ARRAY (L2DistanceApproximate.SIGNATURES), so an UNSUPPORTED + // mapping (the old default-branch behavior) would make the column unusable: + // the distance functions never type-check and the ANN rewrite + // (PushDownVectorTopNIntoPaimonScan) is never even reached. + // + // ARRAY here is Doris's *logical* type; the readers below it do NOT all see a + // plain list, and each needed a VECTOR case of its own: + // - paimon-rust emits Arrow FixedSizeList, not List -- handled in + // DataTypeArraySerDe::read_column_from_arrow. + // - the JNI scanner needs getVector, not getArray, because a vector's binary + // layout differs from an array's -- handled in PaimonColumnValue.unpackArray. + // The declared length is dropped: Doris ARRAY has no fixed-size variant, and + // Paimon enforces the dimension on write. + VectorType vectorType = (VectorType) dataType; + Type vectorElementType = paimonPrimitiveTypeToDorisType(vectorType.getElementType(), + enableVarbinaryMapping, enableTimestampTzMapping); + return org.apache.doris.catalog.ArrayType.create(vectorElementType, true); case MAP: MapType mapType = (MapType) dataType; return new org.apache.doris.catalog.MapType( @@ -423,6 +445,17 @@ public static void updatePaimonColumnUniqueId(Column column, DataType dataType) ArrayType arrayType = (ArrayType) dataType; updatePaimonColumnUniqueId(columns.get(0), arrayType.getElementType()); break; + case VECTOR: + // Mapped to a Doris ARRAY (see paimonPrimitiveTypeToDorisType), so it has the + // same single child column and VectorType delegates collectFieldIds to its + // element exactly as ArrayType does. Falling through to default would leave + // that child's uniqueId at its default instead of the Paimon field id. + // Recursing is safe while VECTOR maps to ARRAY: children.get(0) above is + // unguarded, and the child exists only because Column.createChildrenColumn + // adds a COLUMN_ARRAY_CHILDREN child for any isArrayType(). + VectorType vectorType = (VectorType) dataType; + updatePaimonColumnUniqueId(columns.get(0), vectorType.getElementType()); + break; case MAP: MapType mapType = (MapType) dataType; updatePaimonColumnUniqueId(columns.get(0), mapType.getKeyType()); @@ -495,11 +528,18 @@ public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMap } TNestedField nestedField = new TNestedField(); switch (dataType.getTypeRoot()) { + case VECTOR: case ARRAY: { + // VECTOR shares this branch because paimonPrimitiveTypeToDorisType maps it to + // ARRAY. It must not fall through to the default below: that only sets the + // primitive type and leaves nestedField unset, which would hand BE a field + // declared TPrimitiveType.ARRAY with no item field describing the element. TArrayField listField = new TArrayField(); - org.apache.paimon.types.ArrayType paimonArrayType = (org.apache.paimon.types.ArrayType) dataType; + DataType paimonElementType = dataType.getTypeRoot() == DataTypeRoot.VECTOR + ? ((VectorType) dataType).getElementType() + : ((org.apache.paimon.types.ArrayType) dataType).getElementType(); TFieldPtr fieldPtr = new TFieldPtr(); - fieldPtr.setFieldPtr(getSchemaInfo(paimonArrayType.getElementType(), enableVarbinaryMapping, + fieldPtr.setFieldPtr(getSchemaInfo(paimonElementType, enableVarbinaryMapping, enableTimestampTzMapping)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java index 45b351aa618388..0d25407ed9019c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonPredicateConverter.java @@ -100,6 +100,15 @@ private Predicate doInPredicate(InPredicate predicate) { } String colName = slotRef.getColumnName(); int idx = getFieldIndex(colName); + // Not a column of the Paimon table, so there is no field index to build an IN + // predicate against. Reader-produced synthetic columns land here: the PK-vector + // rewrite turns `dist_fn(vec, q) < x` into a comparison on + // __paimon_search_score_to_dis, which exists only in the scan's output. Returning + // null drops this conjunct from the push-down; Doris still evaluates it itself, so + // the result stays correct -- whereas indexing with -1 threw and killed the query. + if (idx < 0) { + return null; + } DataType dataType = paimonFieldTypes.get(idx); List valueList = new ArrayList<>(); for (int i = 1; i < predicate.getChildren().size(); i++) { @@ -133,6 +142,12 @@ private Predicate binaryExprDesc(Expr dorisExpr) { } String colName = slotRef.getColumnName(); int idx = getFieldIndex(colName); + // Same guard as doInPredicate: a comparison on a column the Paimon table does not + // have (e.g. the reader-produced PK-vector distance column) must be skipped, not + // fatal. Doris still evaluates the conjunct itself. + if (idx < 0) { + return null; + } DataType dataType = paimonFieldTypes.get(idx); Object value = dataType.accept(new PaimonValueConverter(literalExpr)); if (value == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java new file mode 100644 index 00000000000000..95f9f6aa9d300a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustEligibility.java @@ -0,0 +1,319 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon.source; + +import org.apache.doris.datasource.paimon.PaimonScanParams; +import org.apache.doris.system.Backend; + +import org.apache.commons.lang3.StringUtils; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; + +import java.util.Collection; +import java.util.Locale; +import java.util.Map; + +/** + * The table-, storage- and backend-level gates of the paimon-rust reader, + * computed once from the table handle and shared by every call site that must + * agree on them: {@code PaimonScanNode}'s per-split gate (both the batch + * reader and the PK-vector search) and the {@code PushDownVectorTopNIntoPaimonScan} + * rewrite's pre-flight via {@code PaimonVectorSearch.vectorScanEligible}. The + * rewrite is one-way (a rewritten PK-vector split has no JNI fallback), so + * both sites must evaluate the same conditions; this class is that single + * definition. + * + *

Covers only what is a property of the table handle, the storage shape or + * the cluster's backends. Everything that depends on the individual split + * (native DataSplit shape, unmaterialized deletion vectors, external-path + * files, the PaimonRustReaderCapabilities whitelist) or on the final scan + * tuple (ORC TIMESTAMP_LTZ, projected VARIANT) stays with the scan node. + */ +final class PaimonRustEligibility { + + // BE opens the table via paimon_table_from_schema_json, which needs the resolved + // TableSchema that only FileStoreTable exposes via schema(). A table that is not a + // FileStoreTable (e.g. a sys table backed by DataSplit) cannot ship a schema JSON, + // so it falls back to JNI rather than sending an incomplete PAIMON_RUST request + // that BE would reject. + final boolean isFileStoreTable; + + // Both sides of a FallbackReadFileStoreTable wrap their splits, and the pinned + // rust decoder rejects a FallbackDataSplit outright ("trailing bytes after + // DataSplit" — it requires full-buffer consumption); the wrapper is gated as a + // whole (any split from it routes to JNI) until the rust ABI represents both + // sides. The per-split FallbackSplit shape is judged separately in PaimonScanNode. + final boolean fallbackReadTable; + + // A renewable REST token reached the shipped properties as a plain value; only + // the table's FileIO type reveals it expires: doInitialize snapshots + // RESTTokenFileIO.validToken().token() into the backend storage properties, + // discarding expireAtMillis and the REST refresh context, so the shipped + // credentials look static — but the pinned rust table reuses one option map with + // no refresh callback, while the JNI RESTTokenFileIO checks expiry before each + // file operation and obtains a replacement token. A queued or long scan that + // crosses the token TTL would start on rust and later fail authentication. + // Null-safe: a table handle whose FileIO is not resolved stays rust-eligible, + // mirroring the CoreOptions null-safety below. + final boolean restTokenTable; + + // query-auth.enabled tables stay on JNI: when catalog authorization succeeds with + // no row filter or column mask, Paimon still leaves an ordinary DataSplit + // (restricted results use QueryAuthSplit and are already handled by the + // nativeSplit gate), so this table shape passes the compound gate — but the + // shipped schema keeps query-auth.enabled=true and the pinned rust ReadBuilder + // rejects every such table (its CoreOptions::ensure_read_authorized fails closed + // because the client cannot enforce the row filter / column masking), turning a + // valid authorized scan into a BE-open failure. Until the authorization result + // can be transported and enforced by the rust ABI, these tables route to JNI. + final boolean queryAuthTable; + + // Partial-update / aggregation tables with deletion vectors: the pinned rust + // read_pk rejects merge-engine=partial-update/aggregation with + // deletion-vectors.merge-on-read=true outright, so that table option routes the + // whole table to JNI. merge-on-read has no Java accessor in the pinned paimon, so + // it is read raw from the schema options map — the one the BE rust reader + // deserializes from the shipped schema JSON, so it cannot diverge from what BE + // sees — with the rust parsing semantics (any case-insensitive "true" is on, + // default false). For the remaining PU/AGG DV tables, per-split materialization + // is judged in PaimonScanNode (splitDvNotMaterialized); Deduplicate stays + // rust-eligible (its read_pk routes uncompacted splits to the KV reader, which + // applies the attached per-file DVs). + final boolean puAggDeletionVectors; + final boolean dvMergeOnRead; + + // deduplicate.ignore-delete=true tables stay on JNI: Java's + // DeduplicateMergeFunction skips retract records when the option is set — + // including old, uncompacted files that still contain them — but the pinned rust + // read_pk does not pass table options into its deduplicate merge: it picks the + // latest row and omits the key when that row is DELETE/UPDATE_BEFORE. An + // uncompacted insert followed by a delete therefore returns the insert through + // JNI but silently disappears through rust. Gate the option until the rust + // merge implements it. + final boolean deduplicateIgnoreDelete; + + // Non-DV merge options the pinned rust read rejects: Java supports + // partial-update.remove-record-on-delete / aggregation.remove-record-on-delete + // and the wider per-field retract matrix, but the rust PartialUpdateConfig / + // AggregationConfig validations return Unsupported for them — and the DV-derived + // gates above only cover deletion-vector tables, so an ordinary non-DV DataSplit + // with one of these options would pass the compound gate and fail during the + // rust merge construction. Mirrors the exact rust key matrix (presence, not + // values) against the same schema options map BE deserializes. + final boolean rustUnsupportedMergeOption; + + // Branch tables can retain persisted modes without a selector: the shipped + // schema options carry scan.mode, and the pinned Rust builders only accept the + // default. + final boolean scanModeNotDefault; + + // The credential provider shape is evaluated by providerModeTranslatable below. + final boolean providerModeTranslatable; + + // Scheme capability gate: the pinned paimon-rust storage dispatcher (io/storage.rs) + // selects the FileIO parser from the table location's URI scheme, and + // libpaimon_c.a compiles in separate COS, OBS, GCS and Azdls parsers besides the + // OSS and S3 ones. Doris normalizes every object store's credentials into the + // AWS_* / use_path_style aliases, which the BE rust bridge translates only into + // the fs.oss.* and s3.* key families — a cosn:// / obs:// / gs:// / abfs:// + // warehouse would reach its scheme's parser without the key family it reads and + // fail the open instead of using JNI. Only the schemes whose property translation + // is implemented and open-tested (s3 / s3a / oss) plus the credential-free hdfs + // and local-filesystem parsers stay rust-eligible. A null location also routes to + // JNI: the rust path needs the paimon_table that only a real location can provide. + final boolean schemeCapabilityVerified; + + // An hdfs:// location is scheme-verified only together with the credential-free + // backend shape: the backend storage properties that ship to BE also carry an + // HDFS catalog's authentication (kerberos principal / keytab, proxy user, HA + // nameservice config), none of which the pinned rust HDFS parser reads — the + // scan would open as the BE process's ambient identity instead of the catalog's + // configured one and fail the access JNI honors. See isRustVerifiedHdfsBackend. + final boolean hdfsBackendVerified; + + // The BE capability negotiation requires every candidate backend to report + // supports_paimon_rust_reader. + final boolean allBackendsRustCapable; + + // The PK-vector (ANN) protocol is negotiated separately: an older + // rust-capable BE reports the batch bit but would reject an empty + // paimon_split + vector payload ("missing paimon_split"), so the one-way + // vector rewrite must require this bit on every candidate backend too. + final boolean allBackendsVectorCapable; + + private PaimonRustEligibility(boolean isFileStoreTable, boolean fallbackReadTable, + boolean restTokenTable, boolean queryAuthTable, boolean puAggDeletionVectors, + boolean dvMergeOnRead, boolean deduplicateIgnoreDelete, boolean rustUnsupportedMergeOption, + boolean scanModeNotDefault, boolean providerModeTranslatable, boolean schemeCapabilityVerified, + boolean hdfsBackendVerified, boolean allBackendsRustCapable, + boolean allBackendsVectorCapable) { + this.isFileStoreTable = isFileStoreTable; + this.fallbackReadTable = fallbackReadTable; + this.restTokenTable = restTokenTable; + this.queryAuthTable = queryAuthTable; + this.puAggDeletionVectors = puAggDeletionVectors; + this.dvMergeOnRead = dvMergeOnRead; + this.deduplicateIgnoreDelete = deduplicateIgnoreDelete; + this.rustUnsupportedMergeOption = rustUnsupportedMergeOption; + this.scanModeNotDefault = scanModeNotDefault; + this.providerModeTranslatable = providerModeTranslatable; + this.schemeCapabilityVerified = schemeCapabilityVerified; + this.hdfsBackendVerified = hdfsBackendVerified; + this.allBackendsRustCapable = allBackendsRustCapable; + this.allBackendsVectorCapable = allBackendsVectorCapable; + } + + /** + * Whether the credential provider shape in the backend storage properties can be + * translated to the pinned paimon-rust reader without changing the identity the + * catalog configured. The rust S3 bridge maps static credentials, anonymous access + * (AWS_CREDENTIALS_PROVIDER_TYPE=ANONYMOUS -> s3.anonymous) and assume-role, but + * the remaining provider modes are ambient JVM provider chains (ENV, + * SYSTEM_PROPERTIES, WEB_IDENTITY, CONTAINER, INSTANCE_PROFILE) with no paimon-rust + * equivalent — rust would silently sign with whatever its own ambient chain + * resolves to. Lives here as one of the shared eligibility gates; also called + * directly for the JNI fallback-reason logging. + */ + static boolean providerModeTranslatable(String location, + Map backendStorageProperties) { + String providerType = backendStorageProperties == null + ? null : backendStorageProperties.get("AWS_CREDENTIALS_PROVIDER_TYPE"); + String mode = providerType == null ? "DEFAULT" : providerType.trim().toUpperCase(Locale.ROOT); + if (mode.isEmpty()) { + mode = "DEFAULT"; + } + if (location != null && (location.startsWith("s3://") || location.startsWith("s3a://"))) { + // Java DEFAULT may resolve JVM properties or anonymous credentials; + // Rust's ambient chain is different. Only explicit keys or explicit + // anonymous access can cross this boundary without changing identity. + String accessKey = backendStorageProperties == null + ? null : backendStorageProperties.get("AWS_ACCESS_KEY"); + String secretKey = backendStorageProperties == null + ? null : backendStorageProperties.get("AWS_SECRET_KEY"); + boolean staticKeys = accessKey != null && !accessKey.trim().isEmpty() + && secretKey != null && !secretKey.trim().isEmpty(); + // Hadoop's Simple provider gives static keys precedence over anonymous, + // role and token settings. Rust interprets those settings differently. + boolean conflictingStaticSettings = staticKeys && (mode.equals("ANONYMOUS") + || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_ROLE_ARN")) + || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_TOKEN"))); + return !conflictingStaticSettings + && (mode.equals("ANONYMOUS") || (mode.equals("DEFAULT") && staticKeys)); + } + if (providerType != null) { + // OSS has no anonymous FileIO mode in the pinned Rust dependency. + return mode.equals("DEFAULT") + || (mode.equals("ANONYMOUS") && (location == null + || !location.startsWith("oss://"))); + } + return true; + } + + static PaimonRustEligibility of(Table table, String tableLocation, + Map backendStorageProperties, Collection backends) { + boolean isFileStoreTable = table instanceof FileStoreTable; + boolean fallbackReadTable = table instanceof FallbackReadFileStoreTable; + boolean restTokenTable = false; + boolean queryAuthTable = false; + boolean puAggDeletionVectors = false; + boolean dvMergeOnRead = false; + boolean deduplicateIgnoreDelete = false; + boolean rustUnsupportedMergeOption = false; + if (isFileStoreTable) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + restTokenTable = fileStoreTable.fileIO() instanceof RESTTokenFileIO; + CoreOptions resolvedCoreOptions = fileStoreTable.coreOptions(); + // Null-safe: a table handle whose CoreOptions is not resolved (e.g. some + // wrapper shapes) stays rust-eligible rather than failing here — the rust + // open itself rejects such a table if the option is really set. + if (resolvedCoreOptions != null) { + queryAuthTable = resolvedCoreOptions.queryAuthEnabled(); + CoreOptions.MergeEngine mergeEngine = resolvedCoreOptions.mergeEngine(); + if (resolvedCoreOptions.deletionVectorsEnabled() + && (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE + || mergeEngine == CoreOptions.MergeEngine.AGGREGATE)) { + puAggDeletionVectors = true; + TableSchema dvSchema = fileStoreTable.schema(); + Map dvOptions = dvSchema == null ? null : dvSchema.options(); + String mergeOnRead = dvOptions == null + ? null : dvOptions.get(PaimonScanNode.DELETION_VECTORS_MERGE_ON_READ); + dvMergeOnRead = "true".equalsIgnoreCase(mergeOnRead); + } + if (mergeEngine == CoreOptions.MergeEngine.DEDUPLICATE + && resolvedCoreOptions.ignoreDelete()) { + deduplicateIgnoreDelete = true; + } + if (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE + || mergeEngine == CoreOptions.MergeEngine.AGGREGATE) { + TableSchema mergeSchema = fileStoreTable.schema(); + Map mergeOptions = + mergeSchema == null ? null : mergeSchema.options(); + rustUnsupportedMergeOption = mergeOptions != null + && PaimonScanNode.hasRustUnsupportedMergeOption(mergeOptions, mergeEngine); + } + } + } + // Null-safe on a null schema, mirroring the CoreOptions null-safety above: + // a fallback-shaped table can carry a null schema. + boolean scanModeNotDefault = false; + if (isFileStoreTable && ((FileStoreTable) table).schema() != null) { + TableSchema schema = ((FileStoreTable) table).schema(); + String scanMode = PaimonScanParams.withoutTimeTravelSelectors(schema) + .options().get(CoreOptions.SCAN_MODE.key()); + scanModeNotDefault = scanMode != null && !"default".equalsIgnoreCase(scanMode); + } + boolean schemeCapabilityVerified = PaimonScanNode.isRustVerifiedLocationScheme(tableLocation); + boolean hdfsBackendVerified = !PaimonScanNode.isHdfsLocationScheme(tableLocation) + || PaimonScanNode.isRustVerifiedHdfsBackend(backendStorageProperties); + boolean providerModeTranslatable = providerModeTranslatable( + tableLocation, backendStorageProperties); + boolean allBackendsRustCapable = backends != null && !backends.isEmpty() + && backends.stream().allMatch(Backend::isPaimonRustReaderSupported); + boolean allBackendsVectorCapable = backends != null && !backends.isEmpty() + && backends.stream().allMatch(Backend::isPaimonRustVectorReaderSupported); + return new PaimonRustEligibility(isFileStoreTable, fallbackReadTable, restTokenTable, + queryAuthTable, puAggDeletionVectors, dvMergeOnRead, deduplicateIgnoreDelete, + rustUnsupportedMergeOption, scanModeNotDefault, providerModeTranslatable, + schemeCapabilityVerified, hdfsBackendVerified, allBackendsRustCapable, + allBackendsVectorCapable); + } + + /** Whether the table, storage and backend shapes all admit the paimon-rust reader. */ + boolean eligible() { + return isFileStoreTable && !fallbackReadTable + && !restTokenTable && !queryAuthTable + && !dvMergeOnRead && !deduplicateIgnoreDelete && !rustUnsupportedMergeOption + && !scanModeNotDefault + && providerModeTranslatable && schemeCapabilityVerified && hdfsBackendVerified + && allBackendsRustCapable; + } + + /** + * Whether they all admit the PK-vector (ANN) search path on top of the batch + * rust gates: the vector protocol needs every candidate backend to report + * the vector capability bit as well, since a rewritten vector split has no + * JNI fallback and an older rust-capable BE cannot decode it. + */ + boolean vectorEligible() { + return eligible() && allBackendsVectorCapable; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java index d45ff4ac70c3a2..2f42cdc2ddb949 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCapabilities.java @@ -52,23 +52,46 @@ final class PaimonRustReaderCapabilities { private final FileStoreTable table; private final TableSchema schema; private final boolean tableCompatible; + + // Whether this is a primary-key table with the partial-update or aggregation merge + // engine: the engines whose writers historically retained repeated INSERT keys within + // one file, driving the merge reader's unbounded batch retention. + private final boolean puAggPrimaryKeyEngine; private final Map compatibleFileSchemas = new ConcurrentHashMap<>(); PaimonRustReaderCapabilities(FileStoreTable table, TupleDescriptor tuple) { this.table = table; this.schema = table.schema(); this.tableCompatible = schema != null && hasCompatibleSchemaVersion(schema) - && hasFullNestedProjection(tuple) && hasCompatibleMergeEngine(schema) - && hasCompatibleAggregates(schema); + && hasFullNestedProjection(tuple) && hasCompatibleAggregates(schema); + // The merge-engine gate is split-level, not table-level: it exists for the merge + // reader's unbounded losing-batch retention, which only a split that must merge + // can trigger. See canRead. + this.puAggPrimaryKeyEngine = schema != null && hasPuAggPrimaryKeyEngine(schema); } boolean canRead(DataSplit split) { if (!tableCompatible) { return false; } - // The pinned merge reader retains losing input batches until an output batch fills. - // Neither zero deletes nor a small read.batch-size bounds this across multiple files. - if (!schema.primaryKeys().isEmpty() && split.dataFiles().size() > 1) { + // paimon-rust's read_pk dispatch (pk_split_needs_merge / is_fully_materialized_pk_dv): + // a rawConvertible split whose files are all compacted (level != 0) never reaches + // the pinned merge reader. It streams through DataFileReader's raw path instead -- + // one file at a time, each bounded by read.batch-size, with the split's + // deletion-vector factory masking stale rows per file. Both gates below exist + // solely for that merge reader, whose losing-input-batch retention no delete count + // or batch size can bound, so they must not judge a split that never merges. + boolean readsRawWithoutMerge = split.rawConvertible() + && split.dataFiles().stream().allMatch(file -> file.level() != 0); + // Older partial-update/aggregate writers retained repeated INSERT keys within one + // file, so even a single uncompacted file can drive unbounded merge retention. + if (puAggPrimaryKeyEngine && !readsRawWithoutMerge) { + return false; + } + // Across multiple files the same retention accumulates the whole split's inputs + // until an output batch fills. + if (!schema.primaryKeys().isEmpty() && !readsRawWithoutMerge + && split.dataFiles().size() > 1) { return false; } for (DataFileMeta file : split.dataFiles()) { @@ -86,7 +109,60 @@ boolean canRead(DataSplit split) { return true; } - private static boolean hasFullNestedProjection(TupleDescriptor tuple) { + /** Vector-path companion to {@link #canRead}. The vector reader materializes the + * projected rows of its Top-K results through the same Arrow decoders as the batch + * path, so a file whose historical schema version decodes differently must stay off + * the vector path too. Only the per-file schema gates apply: the vector path never + * opens a merge reader (its Top-K chain applies deletion vectors inside the search), + * so the merge-retention gates do not judge it. Loud backstop for shapes the rewrite + * pre-flight cannot see (options-selected schemas, mid-statement ALTER races); it + * mirrors {@link #hasRustIncompatibleSchemaHistory} exactly, so it can only fail + * loudly on shapes the pre-flight could not have declined. */ + boolean canVectorRead(DataSplit split) { + if (schema == null || !hasCompatibleSchemaVersion(schema)) { + return false; + } + for (DataFileMeta file : split.dataFiles()) { + if (file.schemaId() != schema.id() && !compatibleFileSchemas.computeIfAbsent( + file.schemaId(), this::hasCompatibleFileSchema)) { + return false; + } + } + return true; + } + + /** Rewrite pre-flight for the vector path: whether ANY historical schema version of + * the table decodes differently from the current one under the rust reader (the + * per-file gate {@link #canRead} runs per split and cannot judge a whole rewrite). + * Example: a TIMESTAMP(3) column widened to TIMESTAMP(9) while old files remain -- + * the vector reader's safe cast NULLs out-of-range values the ordinary path + * preserves, so the rewrite must decline and let the query run ordinary. + * Conservative when the incompatible files are long gone: rare, and correctness-safe. */ + public static boolean hasRustIncompatibleSchemaHistory(FileStoreTable table) { + TableSchema current = table.schema(); + if (current == null || !hasCompatibleSchemaVersion(current)) { + return true; + } + List history; + try { + history = table.schemaManager().listAll(); + } catch (RuntimeException e) { + // Failure to establish compatibility must not opt a table into Rust. + return true; + } + for (TableSchema historical : history) { + if (historical == null || historical.id() == current.id()) { + continue; + } + if (!hasCompatibleSchemaVersion(historical) + || hasIncompatibleEvolution(historical.fields(), current.fields())) { + return true; + } + } + return false; + } + + static boolean hasFullNestedProjection(TupleDescriptor tuple) { for (SlotDescriptor slot : tuple.getSlots()) { // The ABI projects only root names, while Arrow struct SerDes bind by ordinal. // A pruned slot must use JNI's recursive read type, even inside arrays or maps. @@ -116,12 +192,11 @@ private static boolean hasCompatibleSchemaVersion(TableSchema schema) { return schema.version() > TableSchema.PAIMON_07_VERSION; } - private static boolean hasCompatibleMergeEngine(TableSchema schema) { + private static boolean hasPuAggPrimaryKeyEngine(TableSchema schema) { CoreOptions.MergeEngine engine = new CoreOptions(schema.options()).mergeEngine(); - // Older partial-update/aggregate writers retained repeated INSERT keys within one - // file. Zero deletes cannot bound Rust's retained losing batches for those producers. - return schema.primaryKeys().isEmpty() - || (engine != CoreOptions.MergeEngine.PARTIAL_UPDATE && engine != CoreOptions.MergeEngine.AGGREGATE); + return !schema.primaryKeys().isEmpty() + && (engine == CoreOptions.MergeEngine.PARTIAL_UPDATE + || engine == CoreOptions.MergeEngine.AGGREGATE); } private static boolean hasIncompatibleEvolution(List oldFields, List newFields) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 111bbc905c0281..d95bb861791cbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.paimon.source; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; @@ -53,18 +55,25 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; -import org.apache.doris.system.Backend; import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TExternalSearchQuery; +import org.apache.doris.thrift.TExternalSearchRequest; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TPaimonDeletionFileDesc; import org.apache.doris.thrift.TPaimonFileDesc; import org.apache.doris.thrift.TPaimonReaderType; +import org.apache.doris.thrift.TPaimonVectorPayload; +import org.apache.doris.thrift.TPaimonVectorPayloadType; +import org.apache.doris.thrift.TPaimonVectorSearchOptions; +import org.apache.doris.thrift.TSearchVector; import org.apache.doris.thrift.TTableFormatFileDesc; +import org.apache.doris.thrift.TVectorElementType; +import org.apache.doris.thrift.TVectorMetric; +import org.apache.doris.thrift.TVectorSearchParams; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.paimon.CoreOptions; @@ -72,15 +81,17 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.format.OrcOptions; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataOutputSerializer; import org.apache.paimon.options.Options; +import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.predicate.Predicate; -import org.apache.paimon.rest.RESTTokenFileIO; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.BucketVectorSearchSplit; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.DeletionFile; import org.apache.paimon.table.source.InnerTableScan; @@ -88,6 +99,8 @@ import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.ScanMode; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.table.source.VectorScan; +import org.apache.paimon.table.source.VectorSearchSplit; import org.apache.paimon.table.source.snapshot.SnapshotReader; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataType; @@ -95,8 +108,11 @@ import org.apache.paimon.types.MapType; import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -120,6 +136,10 @@ public class PaimonScanNode extends FileQueryScanNode { private static final long COUNT_WITH_PARALLEL_SPLITS = 10000; private static final long MAX_RETAINED_SERIALIZED_TASK_BYTES = 16L * 1024 * 1024; private volatile PaimonRustReaderCapabilities rustReaderCapabilities; + + // Rust-fallback reason combinations already logged for this scan node, so a + // 960-split scan does not repeat the same one-line reason 960 times. + private Set loggedRustFallbackReasons; private long maxRetainedSerializedTaskBytes = MAX_RETAINED_SERIALIZED_TASK_BYTES; // The keys of incremental read params for Paimon SDK private static final String PAIMON_INCREMENTAL_BETWEEN = "incremental-between"; @@ -142,7 +162,8 @@ public class PaimonScanNode extends FileQueryScanNode { private static final String DORIS_SYSTEM_TABLE_TYPE = "doris.system-table-type"; // Same key as the rust CoreOptions DELETION_VECTORS_MERGE_ON_READ_OPTION: paimon // 1.4 exposes no Java accessor for it, so the raw TableSchema option is read. - private static final String DELETION_VECTORS_MERGE_ON_READ = "deletion-vectors.merge-on-read"; + // Package-private: PaimonVectorSearch.vectorScanEligible reads the same key. + static final String DELETION_VECTORS_MERGE_ON_READ = "deletion-vectors.merge-on-read"; private static final List BACKEND_PAIMON_OPTIONS = Arrays.asList( DORIS_ENABLE_JNI_IO_MANAGER, DORIS_JNI_IO_MANAGER_TMP_DIR, @@ -230,6 +251,17 @@ public String toString() { private Map backendPaimonOptions = Collections.emptyMap(); private Table processedTable; + // Primary-key vector (ANN) top-N info pushed down from Nereids. When set, the + // scan is planned as a vector search (index-only): splits are produced by + // Paimon's newBatchVectorSearchBuilder and BE returns __paimon_search_score_to_dis. + private float[] annSortQueryVector = null; + private String annSortColumnName = null; + private long annSortLimit = -1; + // The index's distance metric (TVectorMetric), validated against the distance + // function by the rewrite rule. BE requires it to convert the score column into + // the distance the function is defined to return. + private TVectorMetric annMetric = null; + // The schema information involved in the current query process (including historical schema). protected ConcurrentHashMap currentQuerySchema = new ConcurrentHashMap<>(); private final Map historyTableSchemas = new ConcurrentHashMap<>(); @@ -338,7 +370,36 @@ public void setSource(PaimonSource source) { protected void convertPredicate() { PaimonPredicateConverter paimonPredicateConverter = new PaimonPredicateConverter( processedTable.rowType()); - predicates = paimonPredicateConverter.convertToPaimonExpr(conjuncts); + predicates = paimonPredicateConverter.convertToPaimonExpr(getPaimonConvertibleConjuncts()); + } + + /** + * The conjuncts worth offering to Paimon, i.e. everything except comparisons on the + * reader-produced PK-vector distance column. + * + *

{@code __paimon_search_score_to_dis} is synthesized by the scan, not read from the + * table: the rewrite rule turns {@code dist_fn(vec, q) < x} into a comparison on this + * slot (PushDownVectorTopNIntoPaimonScan), so a filtered ANN query hands the converter a + * column that no Paimon {@code rowType} contains. The converter itself now returns null + * for unknown columns rather than indexing a field list with -1, so this filter is not + * what keeps the query alive -- it states the intent at the layer that knows about the + * synthetic column, mirroring the exclusion already made for the same column in + * {@code FileQueryScanNode.setColumnPositionMapping}. Dropping the conjunct only skips + * the push-down; Doris still evaluates it, so results are unaffected. + */ + private List getPaimonConvertibleConjuncts() { + List convertible = new ArrayList<>(conjuncts.size()); + for (Expr conjunct : conjuncts) { + List slotRefs = new ArrayList<>(); + conjunct.collect(SlotRef.class, slotRefs); + boolean referencesDistance = slotRefs.stream().anyMatch( + slot -> PaimonVectorSearch.SEARCH_DISTANCE_COLUMN + .equalsIgnoreCase(slot.getColumnName())); + if (!referencesDistance) { + convertible.add(conjunct); + } + } + return convertible; } @Override @@ -377,6 +438,116 @@ public void createScanRangeLocations() throws UserException { String serializedPredicate = PaimonUtil.encodeObjectToString(predicates); params.setPaimonPredicate(serializedPredicate); setScanLevelPaimonOptions(); + // Query-wide vector (ANN) params: set once, shared by every split. + if (isVectorSearch()) { + setExternalSearchRequest(); + } + } + + /** + * Build the provider-independent {@link TExternalSearchRequest} for this Paimon + * PK-vector (ANN) scan and attach it to the scan params (field 39). + * + *

Reuses the community {@code TExternalSearchRequest} / {@code TVectorSearchParams} + * thrift so Paimon and Lance share the same logical search request. Paimon-specific + * index tuning (refine_factor, ivf.nprobe, {@code .metric}, ...) travels in the + * Paimon-only {@link TPaimonVectorSearchOptions} (field 5 of the request); the physical + * per-split bucket payload stays in {@link TPaimonVectorPayload} on {@code TPaimonFileDesc}. + * + *

The query vector is encoded as a {@link TSearchVector} with {@code FLOAT32} element + * type (little-endian), matching paimon-rust's f32 index. The metric enum is passed + * straight through from {@code AnnTopNInfo} with no string conversion. + */ + private void setExternalSearchRequest() { + // Encode the f32 query vector into a little-endian TSearchVector binary. + ByteBuffer buf = ByteBuffer.allocate(annSortQueryVector.length * Float.BYTES) + .order(ByteOrder.LITTLE_ENDIAN); + for (float v : annSortQueryVector) { + buf.putFloat(v); + } + TSearchVector searchVec = new TSearchVector() + .setElementType(TVectorElementType.FLOAT32) + .setDimension(annSortQueryVector.length) + .setValues(buf.array()); + + TVectorSearchParams vectorParams = new TVectorSearchParams() + .setColumn(annSortColumnName) + .setQueryVector(searchVec) + .setTopK(annSortLimit) + // limit already folded with offset by the Nereids rule; BE only needs the + // retrieval count, the real offset is applied by the coordinator TopN. + .setOffset(0) + .setMetric(annMetric); + + TExternalSearchRequest request = new TExternalSearchRequest() + .setSchemaVersion(1) + .setSearchQuery(TExternalSearchQuery.vector_search(vectorParams)); + + // Index/search options. These are query-level *overrides* only: paimon-rust + // already merges the table's own options underneath them, so anything sent + // here that merely repeats the table schema is redundant. + try { + CoreOptions coreOptions = CoreOptions.fromMap(source.getPaimonTable().options()); + String vectorColumn = resolvePaimonFieldName(source.getPaimonTable(), annSortColumnName); + Options indexOptions = coreOptions.primaryKeyVectorIndexOptions(vectorColumn); + if (indexOptions != null) { + Map filtered = filterVectorIndexOptions(indexOptions.toMap(), + vectorColumn, coreOptions.primaryKeyVectorIndexType(vectorColumn)); + request.setPaimonOptions(new TPaimonVectorSearchOptions().setOptions(filtered)); + } + } catch (Exception e) { + LOG.warn("Failed to resolve PK-vector index options for column {}", annSortColumnName, e); + } + params.setExternalSearchRequest(request); + } + + /** + * Keep only the keys that actually configure the vector search, dropping the rest of + * the table's options. + * + *

{@code CoreOptions.primaryKeyVectorIndexOptions} seeds its result with + * {@code new Options(toConfiguration().toMap())}, i.e. a copy of every table + * option, before layering the algorithm-specific ones on top. Forwarding that whole + * map is wrong in three ways: + * + *

    + *
  • It crashes. Catalog audit properties such as {@code owner} / {@code createdBy} + * / {@code updatedBy} can carry a Java null value, which a {@code HashMap} accepts + * and thrift's {@code writeString} does not -- the query dies with an NPE inside + * the vector search options' generated writer before this filter existed.
    • + *
  • It is redundant. paimon-rust merges the table's own options underneath the + * query options itself, so the table half of this map is re-sending what the + * reader already has.
    • + *
  • It over-shares. {@code path}, {@code security.hadoop.username} and any future + * credential-ish property have no business travelling to BE on a search request.
    • + *
+ * + *

The retained prefixes mirror what paimon-rust consumes -- {@code ivf.nprobe} + * (vindex reader), the {@code lumina.} family (stripped by prefix), and + * {@code .metric} -- and match the collection rule in + * {@code CoreOptions.primaryKeyVectorAlgorithmOptions}: per-field options excluding the + * {@code pk-vector.} declarations, plus everything namespaced under the algorithm. + * A null algorithm keeps only the field-scoped half rather than guessing a prefix. + */ + @VisibleForTesting + static Map filterVectorIndexOptions(Map options, + String vectorColumn, String algorithm) { + String fieldPrefix = "fields." + vectorColumn + "."; + String pkVectorPrefix = fieldPrefix + "pk-vector."; + String algorithmPrefix = algorithm == null ? null : algorithm + "."; + Map filtered = new HashMap<>(); + for (Map.Entry entry : options.entrySet()) { + String key = entry.getKey(); + if (key == null || entry.getValue() == null) { + continue; + } + boolean fieldScoped = key.startsWith(fieldPrefix) && !key.startsWith(pkVectorPrefix); + boolean algorithmScoped = algorithmPrefix != null && key.startsWith(algorithmPrefix); + if (fieldScoped || algorithmScoped) { + filtered.put(key, entry.getValue()); + } + } + return filtered; } private void setScanLevelPaimonOptions() { @@ -421,7 +592,8 @@ static boolean isRustVerifiedLocationScheme(String location) { // Whether the location is an hdfs:// table (the only HDFS-family scheme in // RUST_VERIFIED_LOCATION_SCHEMES; viewfs / jfs never pass it). - private static boolean isHdfsLocationScheme(String location) { + // Package-private: PaimonVectorSearch.vectorScanEligible checks the same shape. + static boolean isHdfsLocationScheme(String location) { if (location == null) { return false; } @@ -490,6 +662,27 @@ private static boolean hasFieldOptionSuffix(String key, String suffix) { return key.startsWith("fields.") && key.endsWith(suffix); } + /** + * Whether the pinned paimon-rust vector search can take a DATA predicate on + * this table. Paimon 2.1 permits a PK-vector index on tables without deletion + * vectors (first-row merge engines especially), but the rust + * {@code PkVectorSearchParams::resolve} rejects every data predicate unless + * deletion vectors are enabled AND {@code deletion-vectors.merge-on-read} is + * disabled: its post-recall per-candidate filtering re-reads each candidate + * file's physical rows, which only agrees with the bucket search when the + * table exposes physical rows directly. The options are read raw from the + * table map BE itself deserializes, so the values cannot diverge from what + * the rust reader sees. + */ + public static boolean vectorSearchSupportsDataPredicate(Table paimonTable) { + Map options = paimonTable.options(); + boolean deletionVectors = "true".equalsIgnoreCase( + options.get(CoreOptions.DELETION_VECTORS_ENABLED.key())); + boolean mergeOnRead = "true".equalsIgnoreCase( + options.get(DELETION_VECTORS_MERGE_ON_READ)); + return deletionVectors && !mergeOnRead; + } + // Whether the schema options carry any option key the pinned paimon-rust // read rejects for this merge engine. @VisibleForTesting @@ -757,14 +950,11 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) // pinned rust decoder rejects outright ("trailing bytes after // DataSplit" — it requires full-buffer consumption), and even a // permissive decode would still lack the second table identity - // needed to honor the fallback-side discriminator. Both sides of a - // FallbackReadFileStoreTable wrap their splits, so the table - // wrapper is gated as a whole (any split from it routes to JNI) - // until the rust ABI represents both sides; the FallbackSplit - // interface also catches a wrapper split regardless of how the - // table was resolved here. - boolean fallbackRead = split instanceof FallbackReadFileStoreTable.FallbackSplit - || processedTable instanceof FallbackReadFileStoreTable; + // needed to honor the fallback-side discriminator. The table-half + // of the gate lives in PaimonRustEligibility.fallbackReadTable; the + // FallbackSplit interface here also catches a wrapper split + // regardless of how the table was resolved. + boolean fallbackSplit = split instanceof FallbackReadFileStoreTable.FallbackSplit; // Serialize the same effective table that planning and the JNI reader use. // Relation options such as t@options('read.batch-size'='1') are applied by // getProcessedTable() (doInitialize caches it in processedTable), and the @@ -775,166 +965,23 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) Table paimonTable = processedTable; FileStoreTable paimonFileStoreTable = paimonTable instanceof FileStoreTable ? (FileStoreTable) paimonTable : null; - // query-auth.enabled tables stay on JNI: when catalog authorization - // succeeds with no row filter or column mask, Paimon still leaves an - // ordinary DataSplit (restricted results use QueryAuthSplit and are - // already handled by the nativeSplit gate above), so this table shape - // passes the compound gate — but the shipped schema keeps - // query-auth.enabled=true and the pinned rust ReadBuilder rejects - // every such table (its CoreOptions::ensure_read_authorized fails - // closed because the client cannot enforce the row filter / column - // masking), turning a valid authorized scan into a BE-open failure. - // Until the authorization result can be transported and enforced by - // the rust ABI, these tables route to JNI. - boolean queryAuthTable = false; - // REST-token tables stay on JNI: doInitialize snapshots - // RESTTokenFileIO.validToken().token() into the backend storage - // properties, discarding expireAtMillis and the REST refresh - // context, so the shipped credentials look static — but the - // pinned rust table reuses one option map with no refresh - // callback, while paimon 1.4.2's JNI RESTTokenFileIO checks - // expiry before each file operation and obtains a replacement - // token. A queued or long scan that crosses the token TTL would - // start on rust and later fail authentication. Gate until the - // rust ABI can refresh and atomically update credentials. - boolean restTokenTable = false; - // Partial-update / aggregation tables with deletion vectors only pass - // the rust reader in the fully materialized shape: the pinned rust - // read_pk rejects merge-engine=partial-update/aggregation with - // deletion-vectors.merge-on-read=true outright, and otherwise requires - // every split to be compacted and known free of retract rows - // (DataSplit::is_fully_materialized_pk_dv). Their ordinary DataSplits - // sail through the compound gate above, so without this check a valid - // Java/JNI scan reaches BE and the rust open fails. Deduplicate stays - // rust-eligible: its read_pk routes uncompacted splits to the KV - // reader, which applies the attached per-file DVs. merge-on-read=true - // is a table option, so the whole table routes to JNI; - // non-materialized splits are gated per split below. - boolean puAggDeletionVectors = false; - boolean dvMergeOnRead = false; - boolean deduplicateIgnoreDelete = false; - boolean rustUnsupportedMergeOption = false; - if (paimonFileStoreTable != null) { - // A renewable REST token reached the shipped properties as a - // plain value; only the table's FileIO type reveals it expires. - // Null-safe: a table handle whose FileIO is not resolved stays - // rust-eligible, mirroring the CoreOptions null-safety below. - restTokenTable = paimonFileStoreTable.fileIO() instanceof RESTTokenFileIO; - CoreOptions resolvedCoreOptions = paimonFileStoreTable.coreOptions(); - // Null-safe: a table handle whose CoreOptions is not resolved - // (e.g. some wrapper shapes) stays rust-eligible rather than - // failing the scan here — the rust open itself rejects such a - // table if the option is really set. - if (resolvedCoreOptions != null) { - queryAuthTable = resolvedCoreOptions.queryAuthEnabled(); - CoreOptions.MergeEngine mergeEngine = resolvedCoreOptions.mergeEngine(); - if (resolvedCoreOptions.deletionVectorsEnabled() - && (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE - || mergeEngine == CoreOptions.MergeEngine.AGGREGATE)) { - puAggDeletionVectors = true; - // The merge-engine and deletion-vectors.enabled checks - // above resolve through the Java CoreOptions accessors, - // which the table builds from this same schema options - // map — the one the BE rust reader deserializes from - // the shipped schema JSON — so they cannot diverge from - // what BE sees. merge-on-read has no Java accessor in - // paimon 1.4, so it is read raw from the map, with the - // rust parsing semantics (any case-insensitive "true" - // is on, default false). - TableSchema dvSchema = paimonFileStoreTable.schema(); - Map dvOptions = dvSchema == null ? null : dvSchema.options(); - String mergeOnRead = dvOptions == null - ? null : dvOptions.get(DELETION_VECTORS_MERGE_ON_READ); - dvMergeOnRead = "true".equalsIgnoreCase(mergeOnRead); - } - // deduplicate.ignore-delete=true tables stay on JNI: - // Java's DeduplicateMergeFunction skips retract records - // when the option is set — including old, uncompacted - // files that still contain them — but the pinned rust - // read_pk does not pass table options into its - // deduplicate merge: it picks the latest row and omits - // the key when that row is DELETE/UPDATE_BEFORE. An - // uncompacted insert followed by a delete therefore - // returns the insert through JNI but silently disappears - // through rust. Gate the option until the rust merge - // implements it. - if (mergeEngine == CoreOptions.MergeEngine.DEDUPLICATE - && resolvedCoreOptions.ignoreDelete()) { - deduplicateIgnoreDelete = true; - } - // Non-DV merge options the pinned rust read rejects: Java - // supports partial-update.remove-record-on-delete / - // aggregation.remove-record-on-delete and the wider - // per-field retract matrix, but the rust - // PartialUpdateConfig / AggregationConfig validations - // return Unsupported for them — and the DV-derived gates - // above only cover deletion-vector tables, so an ordinary - // non-DV DataSplit with one of these options would pass the - // compound gate and fail during the rust merge - // construction. Mirror the exact rust key matrix (presence, - // not values) against the same schema options map BE - // deserializes. - if (mergeEngine == CoreOptions.MergeEngine.PARTIAL_UPDATE - || mergeEngine == CoreOptions.MergeEngine.AGGREGATE) { - TableSchema mergeSchema = paimonFileStoreTable.schema(); - Map mergeOptions = - mergeSchema == null ? null : mergeSchema.options(); - rustUnsupportedMergeOption = mergeOptions != null - && hasRustUnsupportedMergeOption(mergeOptions, mergeEngine); - } - } - } - // paimon-rust additionally requires (a) FileScannerV2: the V1 FileScanner + // The table-, storage- and backend-level rust gates are computed once in + // PaimonRustEligibility — the single definition the batch and PK-vector + // branches below share with the PushDownVectorTopNIntoPaimonScan rewrite's + // pre-flight (PaimonVectorSearch.vectorScanEligible) — so both sites agree + // on the same conditions by construction. It is built from processedTable, + // the same effective table serialized below, so relation options such as + // t@options('read.batch-size'='1') reach the gate too. + PaimonRustEligibility rustEligibility = PaimonRustEligibility.of(processedTable, + source.getTableLocation(), backendStorageProperties, + backendPolicy.getBackends()); + // The fallback-read gate halves: the split shape here, the table + // wrapper in rustEligibility.fallbackReadTable. Either routes to JNI. + boolean fallbackRead = fallbackSplit || rustEligibility.fallbackReadTable; + // paimon-rust additionally requires FileScannerV2: the V1 FileScanner // explicitly rejects PAIMON_RUST, so with enable_file_scanner_v2 disabled // the split falls back to JNI instead of encoding a rust request that the - // selected scanner cannot consume, and (b) a FileStoreTable: BE opens the - // table via paimon_table_from_schema_json, which needs the resolved - // TableSchema that only FileStoreTable exposes via schema(). If the table - // is not a FileStoreTable (e.g. a sys table backed by DataSplit), we cannot - // ship a schema JSON, so fall back to CPP / JNI rather than sending an - // incomplete PAIMON_RUST request that BE would reject. - // - // The paimon-rust S3 bridge maps static credentials, anonymous - // access (AWS_CREDENTIALS_PROVIDER_TYPE=ANONYMOUS -> s3.anonymous) - // and assume-role (AWS_ROLE_ARN / AWS_EXTERNAL_ID -> - // s3.assumed.role.*), but the remaining credential-provider modes - // are ambient JVM provider chains (ENV, SYSTEM_PROPERTIES, - // WEB_IDENTITY, CONTAINER, INSTANCE_PROFILE) with no paimon-rust - // equivalent — rust would silently sign with whatever the ambient - // chain resolves to. Gate those modes away from the rust reader - // here so the configured provider is honored via the JNI path. - boolean providerModeTranslatable = true; - String providerType = backendStorageProperties == null - ? null : backendStorageProperties.get("AWS_CREDENTIALS_PROVIDER_TYPE"); - String mode = providerType == null ? "DEFAULT" : providerType.trim().toUpperCase(Locale.ROOT); - if (mode.isEmpty()) { - mode = "DEFAULT"; - } - String location = source.getTableLocation(); - if (location != null && (location.startsWith("s3://") || location.startsWith("s3a://"))) { - // Java DEFAULT may resolve JVM properties or anonymous credentials; - // Rust's ambient chain is different. Only explicit keys or explicit - // anonymous access can cross this boundary without changing identity. - String accessKey = backendStorageProperties == null - ? null : backendStorageProperties.get("AWS_ACCESS_KEY"); - String secretKey = backendStorageProperties == null - ? null : backendStorageProperties.get("AWS_SECRET_KEY"); - boolean staticKeys = accessKey != null && !accessKey.trim().isEmpty() - && secretKey != null && !secretKey.trim().isEmpty(); - // Hadoop's Simple provider gives static keys precedence over anonymous, - // role and token settings. Rust interprets those settings differently. - boolean conflictingStaticSettings = staticKeys && (mode.equals("ANONYMOUS") - || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_ROLE_ARN")) - || StringUtils.isNotEmpty(backendStorageProperties.get("AWS_TOKEN"))); - providerModeTranslatable = !conflictingStaticSettings - && (mode.equals("ANONYMOUS") || (mode.equals("DEFAULT") && staticKeys)); - } else if (providerType != null) { - providerModeTranslatable = mode.equals("DEFAULT") || mode.equals("ANONYMOUS"); - // OSS has no anonymous FileIO mode in the pinned Rust dependency. - if (mode.equals("ANONYMOUS") && location != null && location.startsWith("oss://")) { - providerModeTranslatable = false; - } - } + // selected scanner cannot consume. // Incremental scans (binlog / changelog / delta / diff) must stay // on the JNI path: this wire format carries only an ordinary // DataSplit and the rust reader invokes TableRead::to_arrow, but @@ -982,72 +1029,193 @@ && splitHasOrcFile((DataSplit) split) // rust. Gate until the rust leaf has a Variant Arrow decoder. boolean projectedVariant = desc.getSlots().stream() .anyMatch(slot -> PaimonUtil.containsVariant(slot.getType())); - // Scheme capability gate: the pinned paimon-rust storage - // dispatcher (io/storage.rs) selects the FileIO parser from the - // table location's URI scheme, and libpaimon_c.a compiles in - // separate COS, OBS, GCS and Azdls parsers besides the OSS and S3 - // ones. Doris normalizes every object store's credentials into - // the AWS_* / use_path_style aliases (see the *Properties storage - // classes), which the BE rust bridge translates only into the - // fs.oss.* and s3.* key families — a cosn:// / obs:// / gs:// / - // abfs:// warehouse would reach its scheme's parser without the - // key family it reads (fs.cosn.userinfo.*, fs.obs.*, gcs.*, - // azure.*) and fail the open instead of using JNI. Only the - // schemes whose property translation is implemented and - // open-tested (s3 / s3a / oss, via RUST_VERIFIED_LOCATION_SCHEMES) - // plus the credential-free hdfs and local-filesystem parsers stay - // rust-eligible; every other scheme falls back to JNI. A null - // location also routes to JNI: the rust path needs the - // paimon_table that only a real location can provide (BE rejects - // a split without it). - boolean schemeCapabilityVerified = isRustVerifiedLocationScheme(source.getTableLocation()); - // An hdfs:// location is scheme-verified only together with the credential-free - // backend shape: the backend storage properties that ship to BE also carry an - // HDFS catalog's authentication (kerberos principal / keytab, proxy user, HA - // nameservice config), none of which the pinned rust HDFS parser reads — the - // scan would open as the BE process's ambient identity instead of the - // catalog's configured one and fail the access JNI honors. See - // isRustVerifiedHdfsBackend. - boolean hdfsBackendVerified = !isHdfsLocationScheme(source.getTableLocation()) - || isRustVerifiedHdfsBackend(backendStorageProperties); - // With merge-on-read=true the whole table already routes to JNI (dvMergeOnRead); - // for the remaining partial-update/aggregation DV tables, a split that is + // With merge-on-read=true the whole table already routes to JNI + // (rustEligibility.dvMergeOnRead); for the remaining + // partial-update/aggregation DV tables, a split that is // not fully materialized (uncompacted level-0 data, or retractions not // known to be applied — even a split with no deletion file attached yet) // fails the rust is_fully_materialized_pk_dv guard, so it falls back per // split instead of turning into a BE-open failure. nativeSplit and // !fallbackRead only guard the cast — those splits already route to JNI. - boolean splitDvNotMaterialized = puAggDeletionVectors && !dvMergeOnRead + boolean splitDvNotMaterialized = rustEligibility.puAggDeletionVectors + && !rustEligibility.dvMergeOnRead && nativeSplit && !fallbackRead && !isFullyMaterializedPkDvSplit((DataSplit) split); + boolean vectorSearchSplit = paimonSplit.getVectorPayloadBytes() != null; + // Session, shared table / storage / backend, and scan-shape gates. The + // shared PaimonRustEligibility computation is also the + // PushDownVectorTopNIntoPaimonScan rewrite's pre-flight + // (PaimonVectorSearch.vectorScanEligible): the rewrite is one-way, so both + // sites must agree on the same gates, and the scan node stays authoritative + // — it evaluates the relation-option-processed table, so a t@options + // override the raw-table pre-flight cannot see still rejects here (loudly, + // as the vector path has no JNI fallback). !isIncremental and the + // split-half of the fallback-read gate are the same backstops: an + // incremental scan has no PK-vector semantics (the VectorScan builder has + // no incremental mode), and a FallbackSplit shape cannot ride the vector + // payload either — both fail loudly here rather than silently returning + // snapshot rows for a query that asked for changes. Each gate's value is + // also what the fallback logging below reports. boolean canUseRust = sessionVariable.isEnablePaimonRustReader() - && sessionVariable.enableFileScannerV2 && nativeSplit && !fallbackRead - && !isIncremental && providerModeTranslatable && !queryAuthTable - && !restTokenTable - && !dvMergeOnRead && !splitDvNotMaterialized - && !orcLtzSchema && !projectedVariant && !externalFileSplit - && !deduplicateIgnoreDelete && !rustUnsupportedMergeOption - && schemeCapabilityVerified && hdfsBackendVerified - && paimonFileStoreTable != null; - if (canUseRust) { - // Branch tables can retain persisted modes without a selector. Validate the - // transported options too: the pinned Rust builder only accepts default here. - String scanMode = PaimonScanParams.withoutTimeTravelSelectors(paimonFileStoreTable.schema()) - .options().get(CoreOptions.SCAN_MODE.key()); - canUseRust = scanMode == null || "default".equalsIgnoreCase(scanMode); - } - if (canUseRust) { - // Every candidate can receive this range; older BEs cannot decode reader type 3. - canUseRust = !backendPolicy.getBackends().isEmpty() - && backendPolicy.getBackends().stream().allMatch(Backend::isPaimonRustReaderSupported); + && sessionVariable.enableFileScannerV2 + && rustEligibility.eligible() + && !fallbackRead && !isIncremental + && !orcLtzSchema && !projectedVariant; + // The remaining gates judge the DataSplit the BATCH reader opens: the + // native DataSplit shape, unmaterialized partial-update / aggregation + // deletion vectors, external-path files, and the + // PaimonRustReaderCapabilities whitelist. A PK-vector search split is + // never read as a DataSplit (it rides serialized in vector_payload and + // executes paimon-rust's eager Top-K chain, which applies deletion + // vectors and the merge inside the search itself), so it does not + // consult them. + // + // The nested-projection bound is NOT DataSplit-specific: the ABI + // projects only root names and the Arrow struct SerDe binds children by + // ordinal, so a pruned complex slot (nested pruning, on by default) + // would return the wrong child on the vector path exactly as it would + // on the batch path. The rewrite pre-flights this and declines; this + // is the loud backstop for shapes the raw-table pre-flight cannot see. + boolean vectorSchemaIncompatible = false; + if (vectorSearchSplit) { + canUseRust = canUseRust && rustEligibility.vectorEligible() + && PaimonRustReaderCapabilities.hasFullNestedProjection(desc); + if (canUseRust) { + if (rustReaderCapabilities == null) { + rustReaderCapabilities = new PaimonRustReaderCapabilities(paimonFileStoreTable, desc); + } + // Per-file schema-compat backstop mirroring the rewrite's + // hasRustIncompatibleSchemaHistory pre-flight: the vector + // reader materializes projected rows through the same Arrow + // decoders, so a historically incompatible file must stay off + // the vector path. Can only fail for shapes the pre-flight + // could not see (options-selected schemas, mid-statement + // ALTER races) — loud, like every vector backstop. + vectorSchemaIncompatible = + !rustReaderCapabilities.canVectorRead((DataSplit) split); + canUseRust = canUseRust && !vectorSchemaIncompatible; + } } - if (canUseRust) { - if (rustReaderCapabilities == null) { - rustReaderCapabilities = new PaimonRustReaderCapabilities(paimonFileStoreTable, desc); + if (!vectorSearchSplit && canUseRust) { + canUseRust = canUseRust && nativeSplit && !splitDvNotMaterialized + && !externalFileSplit; + if (canUseRust) { + if (rustReaderCapabilities == null) { + rustReaderCapabilities = new PaimonRustReaderCapabilities(paimonFileStoreTable, desc); + } + canUseRust = rustReaderCapabilities.canRead((DataSplit) split); } - canUseRust = rustReaderCapabilities.canRead((DataSplit) split); } - if (canUseRust) { + // With the rust reader enabled, a logical split that still routes to JNI was + // rejected by a gate above: make the reason visible instead of a silent + // fallback, so "why is this table not using the rust reader" is answerable + // from the FE log. + if (sessionVariable.isEnablePaimonRustReader() && sessionVariable.enableFileScannerV2 + && !canUseRust) { + List reasons = new ArrayList<>(); + if (!vectorSearchSplit && !nativeSplit) { + reasons.add("non_data_split"); + } + if (fallbackRead) { + reasons.add("fallback_read_table"); + } + if (isIncremental) { + reasons.add("incremental_scan"); + } + if (!rustEligibility.providerModeTranslatable) { + reasons.add("provider_mode_untranslatable"); + } + if (rustEligibility.queryAuthTable) { + reasons.add("query_auth_table"); + } + if (rustEligibility.restTokenTable) { + reasons.add("rest_token_table"); + } + if (rustEligibility.dvMergeOnRead) { + reasons.add("dv_merge_on_read"); + } + if (!vectorSearchSplit && splitDvNotMaterialized) { + reasons.add("split_dv_not_materialized"); + } + if (orcLtzSchema) { + reasons.add("orc_ltz_schema"); + } + if (projectedVariant) { + reasons.add("projected_variant"); + } + if (!vectorSearchSplit && externalFileSplit) { + reasons.add("external_file_split"); + } + if (rustEligibility.deduplicateIgnoreDelete) { + reasons.add("deduplicate_ignore_delete"); + } + if (rustEligibility.rustUnsupportedMergeOption) { + reasons.add("unsupported_merge_option"); + } + if (!rustEligibility.schemeCapabilityVerified) { + reasons.add("unverified_location_scheme"); + } + if (!rustEligibility.hdfsBackendVerified) { + reasons.add("hdfs_backend_credentials"); + } + if (rustEligibility.scanModeNotDefault) { + reasons.add("scan_mode_not_default"); + } + if (!rustEligibility.isFileStoreTable) { + reasons.add("not_file_store_table"); + } + if (!rustEligibility.allBackendsRustCapable) { + reasons.add("backend_without_rust_support"); + } + if (vectorSearchSplit && !rustEligibility.allBackendsVectorCapable) { + reasons.add("backend_without_vector_support"); + } + if (vectorSearchSplit + && !PaimonRustReaderCapabilities.hasFullNestedProjection(desc)) { + reasons.add("pruned_nested_projection"); + } + if (vectorSearchSplit && vectorSchemaIncompatible) { + reasons.add("historical_schema_evolution"); + } + if (reasons.isEmpty()) { + reasons.add(vectorSearchSplit ? "unknown" : "capability_whitelist"); + } + String splitDesc = split instanceof DataSplit + ? "bucket=" + ((DataSplit) split).bucket() + + " files=" + ((DataSplit) split).dataFiles().size() + : split.getClass().getSimpleName(); + logRustFallbackReasons(splitDesc, String.join(",", reasons)); + } + if (vectorSearchSplit) { + // Primary-key vector (ANN) search: the serialized bucket split rides in + // vector_payload and BE routes to the paimon-rust vector-search read + // path. The vector path is executed exclusively by the rust reader, so + // a payload with !canUseRust is a broken invariant (the rewrite rule + // gates on both switches): falling back to a plain scan would return an + // empty distance column while every other column holds rows, which is + // silent corruption surfacing as an unattributable size mismatch + // downstream. Fail loudly instead. + if (!canUseRust) { + // RuntimeException because the setScanParams override chain cannot + // declare a checked UserException; the existing native branch throws + // RuntimeException for the same reason. + throw new RuntimeException("Paimon PK-vector search split requires the" + + " paimon-rust reader: enable_file_scanner_v2 must be enabled and" + + " the table must be rust-eligible (storage scheme / credentials," + + " merge options, orc / variant shape)"); + } + TPaimonVectorPayload vectorPayload = new TPaimonVectorPayload(); + vectorPayload.setPayloadType(TPaimonVectorPayloadType.BUCKET_SPLIT_BYTES); + vectorPayload.setBytes(paimonSplit.getVectorPayloadBytes()); + fileDesc.setVectorPayload(vectorPayload); + // The DataSplit is already inside vector_payload (BucketVectorSearchSplit + // embeds it) and the vector read path decodes only vector_payload, never + // paimon_split -- so emit an empty one instead of serializing the same + // split twice on the wire. It must still be set (isset) so BE keeps the + // native-paimon branch rather than rerouting to the orc/parquet reader. + fileDesc.setReaderType(TPaimonReaderType.PAIMON_RUST); + fileDesc.setPaimonSplit(""); + } else if (canUseRust) { fileDesc.setReaderType(TPaimonReaderType.PAIMON_RUST); fileDesc.setPaimonSplit(PaimonUtil.encodeDataSplitToString((DataSplit) split)); } else { @@ -1156,6 +1324,13 @@ protected List getDeleteFiles(TFileRangeDesc rangeDesc) { @Override public List getSplits(int numBackends) throws UserException { + // Primary-key vector (ANN) search is planned entirely by Paimon's + // BatchVectorSearchBuilder: one BucketVectorSearchSplit per (partition, + // bucket), serialized and carried to BE, which runs the whole search + // (recall + refine + row lookup + DV) in a single paimon-rust call. + if (isVectorSearch()) { + return getVectorSearchSplits(); + } boolean forceJniScanner = sessionVariable.isForceJniScanner(); // Paimon system tables need Paimon-side semantics: // - binlog: pack/merge + array materialization @@ -1235,7 +1410,8 @@ public List getSplits(int numBackends) throws UserException { } pushDownCountSplits.add(split); pushDownCountSum += count; - } else if (!forceJniScanner && !forceJniForSystemTable && supportNativeReader(optRawFiles)) { + } else if (!forceJniScanner && !forceJniForSystemTable + && supportNativeReader(optRawFiles)) { // Keep raw-file reads ahead of logical JNI/Rust dispatch: only the native file // reader can use footer aggregates for COUNT(col) and MIN/MAX. Logical DataSplits // may require merge/delete semantics and cannot reuse independent file statistics. @@ -1413,6 +1589,111 @@ public Map getIncrReadParams() throws UserException { return paimonScanParams; } + /** + * Plan a primary-key vector (ANN) search. Produces one Doris split per + * (partition, bucket) {@link BucketVectorSearchSplit}, serialized with the + * native "PKVSPLIT"/v1 protocol so paimon-rust on BE can decode it. Planning + * only needs the vector column and the partition-half of the predicate; the + * query vector and data (residual) predicates are handled by BE. + */ + @VisibleForTesting + public List getVectorSearchSplits() throws UserException { + // The relation may have bound the pinned-empty generation (no commit existed + // at binding). The ordinary path fences that state to an empty result; the + // vector builder would resolve the table's LATEST here and expose a first + // commit that landed after binding, breaking read consistency. Mirror the + // ordinary fence (see getPaimonSplitFromAPI). + Optional relationSnapshot = getRelationSnapshot(); + if (relationSnapshot.isPresent() && relationSnapshot.get() instanceof PaimonMvccSnapshot + && ((PaimonMvccSnapshot) relationSnapshot.get()).getSnapshotCacheValue() + .getSnapshot().getSnapshotId() == PaimonSnapshot.INVALID_SNAPSHOT_ID) { + return Collections.emptyList(); + } + Table paimonTable = getProcessedTable(); + // Only the partition-half of the predicate is pushed into Paimon planning. + // The data (residual) half is left for BE's vector-search filter, and passing + // it here would trip PrimaryKeyVectorScan's DV/merge-on-read pre-filter guard. + PartitionPredicate partitionPredicate = null; + if (predicates != null && !predicates.isEmpty()) { + Pair, List> partSplit = + PartitionPredicate.splitPartitionPredicatesAndDataPredicates( + predicates, paimonTable.rowType(), paimonTable.partitionKeys()); + partitionPredicate = partSplit.getLeft().orElse(null); + } + + // Resolve the actual Paimon field name (case-insensitive); the pushed-down + // column name comes from the Doris column, which may differ in case. + String vectorColumn = resolvePaimonFieldName(paimonTable, annSortColumnName); + + VectorScan.Plan plan; + try { + plan = paimonTable.newBatchVectorSearchBuilder() + .withVectorColumn(vectorColumn) + .withPartitionFilter(partitionPredicate) + .newVectorScan() + .scan(); + } catch (Exception e) { + throw new UserException("Paimon PK-vector search planning failed: " + e.getMessage(), e); + } + + List splits = new ArrayList<>(); + Set partitions = new HashSet<>(); + // Same partition-metadata requirement as the ordinary split loop below: a + // table with partition keys makes setPaimonParams call setPartitionValues, + // which throws "Missing partition values" on a null map, so every vector + // split over a partitioned table must carry the derived values too. + boolean needPartitionMetadata = !getOrderedPathPartitionKeys().isEmpty(); + Map> partitionInfoMaps = new HashMap<>(); + for (VectorSearchSplit vectorSplit : plan.splits()) { + if (!(vectorSplit instanceof BucketVectorSearchSplit)) { + throw new UserException("Unexpected PK-vector split type: " + + vectorSplit.getClass().getName()); + } + BucketVectorSearchSplit bucketSplit = (BucketVectorSearchSplit) vectorSplit; + // Serialize with Paimon's own DataOutputSerializer: the bytes must be + // exactly BucketVectorSearchSplit.serialize's DataOutputView form + // (magic "PKVSPLIT", version 1, big-endian), which + // paimon_bucket_vector_search_split_deserialize on BE decodes. This + // repository's Paimon no longer ships a BucketVectorSearchSplitSerializer. + byte[] payloadBytes; + try { + DataOutputSerializer output = new DataOutputSerializer(256); + bucketSplit.serialize(output); + payloadBytes = output.getCopyOfBuffer(); + } catch (IOException e) { + throw new UserException("Failed to serialize PK-vector split: " + e.getMessage(), e); + } + DataSplit dataSplit = bucketSplit.dataSplit(); + PaimonSplit split = new PaimonSplit(dataSplit); + split.setVectorPayloadBytes(payloadBytes); + if (needPartitionMetadata) { + Map partitionInfoMap = + partitionInfoMaps.computeIfAbsent(dataSplit.partition(), k -> PaimonUtil.getPartitionInfoMap( + paimonTable, k, sessionVariable.getTimeZone(), + source.getCatalog().getEnableMappingTimestampTz())); + split.setPaimonPartitionValues(partitionInfoMap); + } + splits.add(split); + partitions.add(dataSplit.partition()); + ++paimonSplitNum; + } + + splits.forEach(s -> s.setTargetSplitSize(sessionVariable.getFileSplitSize() > 0 + ? sessionVariable.getFileSplitSize() : sessionVariable.getMaxSplitSize())); + this.selectedPartitionNum = partitions.size(); + return splits; + } + + /** Resolve a Paimon row-type field name case-insensitively, else return the input. */ + private static String resolvePaimonFieldName(Table paimonTable, String name) { + for (String field : paimonTable.rowType().getFieldNames()) { + if (field.equalsIgnoreCase(name)) { + return field; + } + } + return name; + } + @VisibleForTesting public List getPaimonSplitFromAPI() throws UserException { long startTime = System.currentTimeMillis(); @@ -1780,6 +2061,23 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { sb.append(String.format("%spaimonNativeReadSplits=%d/%d\n", prefix, rawFileSplitNum, (paimonSplitNum + rawFileSplitNum))); + if (annSortQueryVector != null) { + sb.append(prefix).append("annSortQueryVector=") + .append(Arrays.toString(annSortQueryVector)).append("\n"); + } + if (annSortColumnName != null) { + sb.append(prefix).append("annSortColumnName=").append(annSortColumnName).append("\n"); + } + // Worth showing even though it is not a plan choice: it selects the score->distance + // transform the BE reader applies, so an ANN plan that looks right can still return + // wrong distances if this is not what was expected. + if (annMetric != null) { + sb.append(prefix).append("annMetric=").append(annMetric).append("\n"); + } + if (annSortLimit != -1) { + sb.append(prefix).append("annSortLimit=").append(annSortLimit).append("\n"); + } + sb.append(prefix).append("predicatesFromPaimon:"); if (predicates.isEmpty()) { sb.append(" NONE\n"); @@ -2052,4 +2350,63 @@ private Table getProcessedTable() throws UserException { } return finalTable; } + + public void setAnnSortQueryVector(float[] annSortQueryVector) { + this.annSortQueryVector = annSortQueryVector; + } + + public void setAnnSortColumnName(String annSortColumnName) { + this.annSortColumnName = annSortColumnName; + } + + public void setAnnSortLimit(long annSortLimit) { + this.annSortLimit = annSortLimit; + } + + public void setAnnMetric(TVectorMetric annMetric) { + this.annMetric = annMetric; + } + + /** + * Whether this scan is planned as a primary-key vector (ANN) search. Requires + * the query vector, indexed column and retrieval limit to all be present. + * + *

Deliberately does NOT also require the metric, and adding that check would + * make things worse rather than safer. This predicate is a control switch, not a + * query: it decides whether {@link #getSplits} plans bucket-vector splits and + * whether {@code external_search_request} is sent at all. But whether the scan carries + * a distance column was already decided earlier, by + * PushDownVectorTopNIntoPaimonScan, which replaced the distance expression with the + * {@code __paimon_search_score_to_dis} slot and put it in the scan output -- that is + * not undoable here. Returning false on a missing metric would therefore plan a + * plain scan whose projection excludes that column, leaving it empty while every other + * column holds num_rows: silent corruption surfacing downstream as an unattributable + * size mismatch. + * + *

The check is also unreachable: the only AnnTopNInfo construction site passes a + * {@code "l2"}/{@code "inner_product"} literal and bails out (no rewrite at all) + * when the index metric disagrees. And if it ever did leak through, BE already + * rejects an unset or unsupported metric with an explicit error -- a loud failure, + * which is strictly better than a silently degraded plan. + */ + /** + * Log why a logical paimon split routed to JNI although the rust reader was enabled. + * One line per distinct reason combination per scan node, so a 960-split scan whose + * splits fail the same gate logs once, while a scan with differently-shaped splits + * (e.g. per-split DV materialization) logs each distinct shape once. + */ + private void logRustFallbackReasons(String splitDesc, String reasons) { + if (loggedRustFallbackReasons == null) { + loggedRustFallbackReasons = new HashSet<>(); + } + if (loggedRustFallbackReasons.add(reasons)) { + LOG.info("paimon-rust reader enabled but a logical paimon split routed to JNI:" + + " table={} split={} reasons=[{}]", + source == null ? "unknown" : source.getTableLocation(), splitDesc, reasons); + } + } + + public boolean isVectorSearch() { + return annSortQueryVector != null && annSortColumnName != null && annSortLimit >= 0; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java index 4a8808517b2176..b130fdc726ad0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java @@ -42,6 +42,10 @@ public class PaimonSplit extends FileSplit { private Optional optRowCount = Optional.empty(); private Optional schemaId = Optional.empty(); private Map paimonPartitionValues = null; + // Serialized BucketVectorSearchSplit (magic "PKVSPLIT", version 1, big-endian) for a + // primary-key vector (ANN) search split. Non-null only in vector mode; BE routes to the + // paimon-rust vector-search read path when this payload is present. + private byte[] vectorPayloadBytes = null; /** * Constructor for Paimon splits. @@ -137,6 +141,14 @@ public Map getPaimonPartitionValues() { return paimonPartitionValues; } + public void setVectorPayloadBytes(byte[] vectorPayloadBytes) { + this.vectorPayloadBytes = vectorPayloadBytes; + } + + public byte[] getVectorPayloadBytes() { + return vectorPayloadBytes; + } + public static class PaimonSplitCreator implements SplitCreator { static final PaimonSplitCreator DEFAULT = new PaimonSplitCreator(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java new file mode 100644 index 00000000000000..eba89e04536bc2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonVectorSearch.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon.source; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Type; +import org.apache.doris.system.Backend; + +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; + +import java.util.Collection; +import java.util.Map; + +/** + * Shared definition of the reader-produced primary-key vector (ANN) search output + * column for the index-only (Approach B) search path. + * + *

The column holds a distance, not paimon-rust's score. paimon-rust emits + * an Arrow field named {@code __paimon_search_score} carrying a higher-is-better + * score, and the BE PaimonRustTableReader inverts that transform in place while + * filling the block, so the value Doris sees is exactly what + * {@code l2_distance_approximate} / {@code inner_product_approximate} are defined + * to return. + * + *

That is why the Doris-side name is {@link #SEARCH_DISTANCE_COLUMN} + * ({@code __paimon_search_score_to_dis}) and not the Arrow name: the two differ in + * meaning, so they must not share a name. Keeping them distinct also leaves + * {@code __paimon_search_score} free should a later version want to expose the raw + * score as its own column. BE bridges the two names explicitly when filling the + * block; the Arrow-side name stays owned by paimon-rust. Unlike Doris hidden columns + * this one deliberately does not carry the {@code __DORIS_} prefix. + * + *

Because the column is reader-produced (not a real Paimon table column), its + * slot must still carry a non-null {@link Column} so it survives + * {@code FileQueryScanNode.initSchemaParams}, but it is excluded from the Parquet/ORC + * column-position mapping (see {@code FileQueryScanNode.setColumnPositionMapping}). + */ +public final class PaimonVectorSearch { + + /** + * Doris-side name of the reader-produced distance column. Must match + * {@code kPaimonSearchDistanceColumn} in the BE reader, which maps + * paimon-rust's {@code __paimon_search_score} Arrow field onto it. + */ + public static final String SEARCH_DISTANCE_COLUMN = "__paimon_search_score_to_dis"; + + private static final Column DISTANCE_COLUMN = createDistanceColumn(); + + private PaimonVectorSearch() {} + + /** Shared, immutable distance column instance; do not mutate. */ + public static Column getDistanceColumn() { + return DISTANCE_COLUMN; + } + + private static Column createDistanceColumn() { + // Non-nullable FLOAT, matching the distance slot produced by the rewrite rule. + Column column = new Column(SEARCH_DISTANCE_COLUMN, Type.FLOAT, false, + "Paimon primary-key vector search distance (reader-produced)"); + column.setIsVisible(false); + return column; + } + + /** + * Whether a PK-vector (ANN) top-N on this table can execute on the paimon-rust + * vector search path. This is the single definition shared by the + * {@code PushDownVectorTopNIntoPaimonScan} rewrite's pre-flight and + * {@code PaimonScanNode}'s per-split vector gate, because the rewrite is one-way: + * a rewritten vector split has no JNI fallback (the JNI reader ignores the vector + * payload and would return a non-ANN scan with an empty distance column), so the + * scan node can only fail loudly — a gate-failing table must therefore not be + * rewritten at all, leaving the query to run as an ordinary scan that computes the + * distance in Doris. + * + *

Delegates to {@link PaimonRustEligibility}, the single computation of + * the table-, storage- and backend-level gates also consumed by the batch + * rust gate in {@code PaimonScanNode}. Does not cover the + * projection-dependent bounds (ORC TIMESTAMP_LTZ, projected VARIANT): they + * depend on the final scan tuple and stay scan-node-side. The scan node + * evaluates the relation-option-processed table while the rewrite sees the + * raw cached table, so a {@code t@options} override can still make the + * scan node reject a split the pre-flight accepted — the scan node stays + * authoritative and fails loudly. The BE capability negotiation requires + * every candidate backend to report {@code supports_paimon_rust_reader} AND + * {@code supports_paimon_rust_vector_reader}: the vector protocol rides an + * empty {@code paimon_split} plus a vector payload, which an older + * rust-capable BE (rolling upgrade) rejects as "missing paimon_split", so + * the vector bit is negotiated separately from the batch rust bit. + */ + public static boolean vectorScanEligible(Table paimonTable, String tableLocation, + Map backendStorageProperties, Collection backends) { + return PaimonRustEligibility.of(paimonTable, tableLocation, + backendStorageProperties, backends).vectorEligible(); + } + + /** + * Rewrite pre-flight facade for + * {@link PaimonRustReaderCapabilities#hasRustIncompatibleSchemaHistory}: the + * vector reader materializes projected values through Arrow safe casts, which + * can silently NULL out-of-range values on a historically evolved schema (a + * TIMESTAMP(3) -> TIMESTAMP(9) widening reading year-2300 data); the rewrite + * must decline such tables so the query runs through the ordinary path. + */ + public static boolean hasRustIncompatibleSchemaHistory(FileStoreTable table) { + return PaimonRustReaderCapabilities.hasRustIncompatibleSchemaHistory(table); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java index 89dea6d31070b0..bddddf4e05e2df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java @@ -60,6 +60,7 @@ public abstract class AbstractPaimonProperties extends MetastoreProperties { private static final String DORIS_JNI_PROPERTY_PREFIX = "paimon.jni."; /** The suffix after this prefix is passed to Paimon as a dynamic table option. */ public static final String TABLE_OPTION_PREFIX = PaimonReaderOptions.TABLE_OPTION_PREFIX; + private static final String HADOOP_S3A_PREFIX = "fs.s3a."; private Map tableOptionsMap = Collections.emptyMap(); @@ -123,6 +124,34 @@ protected void appendUserHadoopConfig(Configuration conf) { normalizeS3Config().forEach(conf::set); } + /** + * Mirrors the S3 credentials Doris assembled into the Hadoop configuration + * onto the catalog options, so the pinned Paimon 2.1 SDK sees them. + * + *

Doris stores a catalog's storage credentials (fs.s3a.access.key, fs.s3a.secret.key, + * fs.s3a.endpoint, ...) only in the Hadoop Configuration handed to + * CatalogContext.create(options, conf) — the paimon 1.4 FileIO consulted exactly + * that conf. The pinned 2.1 S3FileIO instead reads its credentials exclusively from + * CatalogContext.options() (S3FileIO#configure -> loadHadoopConfigFromContext copies + * the s3./s3a./fs.s3a.-prefixed option entries and never looks at the conf), so + * without this mirror every s3-warehouse catalog initializes the SDK with no + * credentials and the first file operation fails on the ambient AWS provider chain + * ("Unable to load credentials from system settings"). Mirroring the whole fs.s3a.* + * set keeps user extras (path style, connection limits) flowing as well; entries + * outside the prefix are not the SDK's concern. + */ + protected void mirrorHadoopS3OptionsToCatalogOptions(Configuration conf) { + if (conf == null || catalogOptions == null) { + return; + } + conf.forEach(entry -> { + String key = entry.getKey(); + if (key.startsWith(HADOOP_S3A_PREFIX) && StringUtils.isNotBlank(entry.getValue())) { + catalogOptions.set(key, entry.getValue()); + } + }); + } + public Map getCatalogOptionsMap() { // Return the cached map if already initialized Map existing = catalogOptionsMapRef.get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java index df0ebae97490a8..78daa9ed498b23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java @@ -48,6 +48,7 @@ public Catalog initializeCatalog(String catalogName, List sto } }); appendUserHadoopConfig(conf); + mirrorHadoopS3OptionsToCatalogOptions(conf); CatalogContext catalogContext = CatalogContext.create(catalogOptions, conf); try { return this.executionAuthenticator.execute(() -> CatalogFactory.createCatalog(catalogContext)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java index e7e6689d3e3cab..56756f563d2c96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java @@ -90,6 +90,7 @@ public Catalog initializeCatalog(String catalogName, List sto buildCatalogOptions(); Configuration conf = buildHiveConfiguration(storagePropertiesList); appendUserHadoopConfig(conf); + mirrorHadoopS3OptionsToCatalogOptions(conf); CatalogContext catalogContext = CatalogContext.create(catalogOptions, conf); try { return executionAuthenticator.execute(() -> CatalogFactory.createCatalog(catalogContext)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java index 7568d59c5fed33..bd4f8876d84e30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java @@ -122,6 +122,7 @@ public Catalog initializeCatalog(String catalogName, List sto } } appendUserHadoopConfig(conf); + mirrorHadoopS3OptionsToCatalogOptions(conf); if (StringUtils.isNotBlank(driverUrl)) { registerJdbcDriver(driverUrl, driverClass); LOG.info("Using dynamic JDBC driver for Paimon JDBC catalog from: {}", driverUrl); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 527442b63632a7..4793097a1fc186 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -802,6 +802,14 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla } else if (table.getType() == TableIf.TableType.PAIMON_EXTERNAL_TABLE) { scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv, context.getScanContext()); + // Translate primary-key vector (ANN) top-N info pushed down onto the scan. + fileScan.getAnnTopN().ifPresent(ann -> { + PaimonScanNode paimonScanNode = (PaimonScanNode) scanNode; + paimonScanNode.setAnnSortQueryVector(ann.getQueryVector()); + paimonScanNode.setAnnSortColumnName(ann.getColumnName()); + paimonScanNode.setAnnSortLimit(ann.getLimit()); + paimonScanNode.setAnnMetric(ann.getMetric()); + }); } else if (table instanceof TrinoConnectorExternalTable) { scanNode = new TrinoConnectorScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv, context.getScanContext()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 9167a0aed681dd..47e00612097684 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -148,6 +148,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushDownUnnestInProject; import org.apache.doris.nereids.rules.rewrite.PushDownVectorTopNIntoOlapScan; +import org.apache.doris.nereids.rules.rewrite.PushDownVectorTopNIntoPaimonScan; import org.apache.doris.nereids.rules.rewrite.PushDownVirtualColumnsIntoOlapScan; import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin; import org.apache.doris.nereids.rules.rewrite.PushProjectIntoUnion; @@ -791,6 +792,7 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown(DistinctAggregateRewriter.INSTANCE), custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, EliminateUnnecessaryProject::new), topDown(new PushDownVectorTopNIntoOlapScan()), + topDown(new PushDownVectorTopNIntoPaimonScan()), topDown(new PushDownVirtualColumnsIntoOlapScan()), topDown(new PushDownMatchProjectionAsVirtualColumn()), topic("score optimize", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 25f46eba8e563c..dffe4ad400c82b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -346,6 +346,7 @@ public enum RuleType { PUSH_DOWN_MATCH_PROJECTION_AS_VIRTUAL_COLUMN(RuleTypeClass.REWRITE), PUSH_DOWN_SCORE_TOPN_INTO_OLAP_SCAN(RuleTypeClass.REWRITE), PUSH_DOWN_VECTOR_TOPN_INTO_OLAP_SCAN(RuleTypeClass.REWRITE), + PUSH_DOWN_VECTOR_TOPN_INTO_PAIMON_SCAN(RuleTypeClass.REWRITE), CHECK_SCORE_USAGE(RuleTypeClass.REWRITE), OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE), PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java index c53e07d86c973e..dc6acdd0e77a14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java @@ -44,7 +44,8 @@ public Rule build() { fileScan.getTableSnapshot(), fileScan.getOperativeSlots(), fileScan.getScanParams(), - fileScan.getRelationSnapshot()) + fileScan.getRelationSnapshot(), + fileScan.getAnnTopN()) ).toRule(RuleType.LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java new file mode 100644 index 00000000000000..c76c152f814ba6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScan.java @@ -0,0 +1,756 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.credentials.CredentialUtils; +import org.apache.doris.datasource.credentials.VendedCredentialsFactory; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonUtil; +import org.apache.doris.datasource.paimon.source.PaimonScanNode; +import org.apache.doris.datasource.paimon.source.PaimonVectorSearch; +import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProductApproximate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.plans.AnnTopNInfo; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TVectorMetric; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Push a primary-key vector (ANN) top-N down into the Paimon scan (index-only). + * + *

Recognizes {@code TopN[order by dis] -> Project[dis := dist_fn(vec, q)] -> [Filter] -> PaimonScan} + * where {@code dist_fn} is {@code l2_distance_approximate} (ASC) or + * {@code inner_product_approximate} (DESC) and the Paimon table has a PK-vector + * index on {@code vec} with a matching distance metric. + * + *

The rewrite mirrors the internal-table ANN path (see + * {@code PushDownVectorTopNIntoOlapScan}): the scan is told to produce a + * reader-generated {@code __paimon_search_score_to_dis} column via the PK-vector index, and + * every occurrence of {@code dist_fn(vec, q)} is replaced by that column's slot. The + * order key slot keeps its identity, so the surrounding TopN merges/limits unchanged. + * The distance always comes from the index, never from the raw vector. + * + *

No arithmetic is emitted here even though paimon-rust reports a score rather than + * a distance: the BE reader undoes the metric-dependent score transform in place while + * filling the block, exactly as the internal table sqrt's faiss's squared distance inside + * the index layer. So the column arrives already holding what {@code dist_fn} would + * return, and the plan stays a bare slot reference -- which also keeps {@code dis} + * non-nullable (an expression over the score would not be, since {@code Divide} and + * {@code Sqrt} are always nullable) and makes predicates like + * {@code dist_fn(vec, q) < 10} correct without any direction flip. + * + *

Being index-only is just an optimization here, not a precondition: if nothing + * else references the raw vector column it is left unreferenced and dropped by column + * pruning (pure index-only read); if the query still needs it ({@code SELECT vec} or a + * predicate on {@code vec}), it stays a normal projected column and the BE vector + * reader materializes it alongside the score. Either way ANN is applied -- the rewrite + * never falls back merely because the vector is read. + */ +public class PushDownVectorTopNIntoPaimonScan implements RewriteRuleFactory { + + /** Doris column types the BE paimon-rust predicate converter can push; see + * PaimonRustPredicateConverter._is_supported_slot_type on the BE side. */ + private static final Set VECTOR_FILTER_SUPPORTED_COLUMN_TYPES = ImmutableSet.of( + Type.BOOLEAN, Type.TINYINT, Type.SMALLINT, Type.INT, Type.BIGINT, + Type.STRING, Type.DATE, Type.DATEV2); + + /** The BE rust predicate converter encodes the IN option list under a 16-bit + * child count; a bigger list is dropped whole into the post-fetch residual. */ + private static final int MAX_IN_PREDICATE_CHILDREN = 0xFFFF; + + @Override + public List buildRules() { + return ImmutableList.of( + logicalTopN(logicalProject(logicalFileScan())).when(t -> t.getOrderKeys().size() == 1).then(topN -> { + LogicalProject project = topN.child(); + LogicalFileScan scan = project.child(); + return pushDown(topN, project, scan, Optional.empty()); + }).toRule(RuleType.PUSH_DOWN_VECTOR_TOPN_INTO_PAIMON_SCAN), + logicalTopN(logicalProject(logicalFilter(logicalFileScan()))) + .when(t -> t.getOrderKeys().size() == 1).then(topN -> { + LogicalProject> project = topN.child(); + LogicalFilter filter = project.child(); + LogicalFileScan scan = filter.child(); + return pushDown(topN, project, scan, Optional.of(filter)); + }).toRule(RuleType.PUSH_DOWN_VECTOR_TOPN_INTO_PAIMON_SCAN) + ); + } + + private Plan pushDown( + LogicalTopN topN, + LogicalProject project, + LogicalFileScan scan, + Optional> optionalFilter) { + // Only Paimon external tables carry PK-vector indexes. + if (!(scan.getTable() instanceof PaimonExternalTable)) { + return null; + } + + // The index-only search path is executed exclusively by the BE paimon-rust reader, + // which only FileScannerV2 can select (the V1 FileScanner explicitly rejects + // PAIMON_RUST). If the rust reader or the V2 scanner is disabled, the split would + // fall through to the JNI reader, which ignores the vector payload and would + // return a non-ANN scan with an empty distance column. Fall back to a normal + // scan then. + ConnectContext ctx = ConnectContext.get(); + if (ctx == null || ctx.getSessionVariable() == null + || !ctx.getSessionVariable().isEnablePaimonRustReader() + || !ctx.getSessionVariable().enableFileScannerV2) { + return null; + } + + // An incremental scan (t@incr / BETWEEN) must not be rewritten into a + // PK-vector search: the VectorScan builder has no incremental mode, so a + // vector payload would silently return snapshot rows for a query that asked + // for changes. Keeping the query un-rewritten runs it as the ordinary + // incremental scan the user requested, with the distance computed in Doris. + // (The scan node's vector gate keeps !isIncremental as the authoritative + // backstop for any path that produces a vector split regardless.) + // + // The same holds for a pinned file-creation-time read state: the ordinary + // path enforces `scan.file-creation-time-millis` by dropping manifest + // entries older than the bound, but the vector search plans its own splits + // and never consults that bound, so old files' nearest neighbors could leak + // into the result. Snapshot/tag/version selectors are fine -- they are + // resolved into the table handle before the vector search -- so only the + // file-level bound forces the decline. + // + // `scan.creation-time-millis` must decline as well: when a snapshot exists + // past the bound it resolves into an ordinary snapshot pin (safe), but when + // none does — the exact read state the option exists for — PaimonScanParams + // resolves it into the SAME internal file-creation bound (resolveOptions + // falls through to resolveFileCreationTime), which the vector path again + // never consults. Which case applies depends on the snapshot manager's + // state, so the raw option key is declined unconditionally. + if (scan.getScanParams().isPresent()) { + TableScanParams scanParams = scan.getScanParams().get(); + if (scanParams.incrementalRead()) { + return null; + } + Map rawParams = scanParams.getMapParams(); + if (rawParams.containsKey(CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key()) + || rawParams.containsKey(CoreOptions.SCAN_CREATION_TIME_MILLIS.key())) { + return null; + } + } + + // A bounded vector top-N is only correct when every filter conjunct is + // applied inside the search, before the limit is exhausted. The BE + // paimon-rust predicate converter skips conjuncts it cannot represent (and + // the synthetic distance column is not in its field registry at all), and + // the engine re-applies those residuals on the returned block -- after the + // top-K rows were already selected. A skipped residual can therefore drop + // rows out of a bounded result: with candidate distances 0, 2 and 3, a + // `WHERE dis > 1 ORDER BY dis LIMIT 1` would fetch only the distance-0 row, + // filter it out and return nothing instead of the distance-2 row. So the + // rewrite only fires when each conjunct is provably of a form the converter + // pushes: a simple comparison / IN / null-check on a table column whose + // Doris type is in the converter's supported set (see + // PaimonRustPredicateConverter._is_supported_slot_type on BE). Anything + // else -- an expression predicate, a predicate on the distance itself, an + // unsupported column type -- declines the rewrite and the query runs as an + // ordinary scan with the distance computed in Doris, which is always + // correct. + if (optionalFilter.isPresent() && !conjunctsRideIntoVectorSearch(optionalFilter.get())) { + return null; + } + + // The order key must be a SlotReference produced by a Project alias. + Expression orderKey = topN.getOrderKeys().get(0).getExpr(); + if (!(orderKey instanceof SlotReference)) { + return null; + } + SlotReference keySlot = (SlotReference) orderKey; + Alias orderKeyAlias = null; + Expression orderKeyExpr = null; + for (NamedExpression projection : project.getProjects()) { + if (projection.toSlot().equals(keySlot) && projection instanceof Alias) { + orderKeyAlias = (Alias) projection; + orderKeyExpr = orderKeyAlias.child(); + break; + } + } + if (orderKeyExpr == null) { + return null; + } + + // l2 must sort ascending (smaller distance = closer); inner_product descending. + boolean isAsc = topN.getOrderKeys().get(0).isAsc(); + boolean l2Dist = orderKeyExpr instanceof L2DistanceApproximate; + boolean innerProduct = orderKeyExpr instanceof InnerProductApproximate; + if (!(l2Dist && isAsc) && !(innerProduct && !isAsc)) { + return null; + } + + // The indexed column must be a direct slot reference. A Cast chain over the + // column is NOT stripped: a legal cast can change vector values (for example + // an ARRAY -> ARRAY -> ARRAY round trip loses precision), + // and searching the original vectors while the query means the casted ones + // changes both the distance and the row order. Doris's implicit coercions do + // not produce a cast here for a matching ARRAY column, so anything + // wrapped in a Cast simply declines the rewrite. + Expression left; + Expression right; + if (l2Dist) { + left = ((L2DistanceApproximate) orderKeyExpr).left(); + right = ((L2DistanceApproximate) orderKeyExpr).right(); + } else { + left = ((InnerProductApproximate) orderKeyExpr).left(); + right = ((InnerProductApproximate) orderKeyExpr).right(); + } + if (!(left instanceof SlotReference && right.isConstant() && right instanceof ArrayLiteral)) { + return null; + } + // A typed NULL inside the query vector ([CAST(NULL AS FLOAT), 0.0, ...]) is a + // NullLiteral whose getDouble() returns 0, so extracting the coordinates + // would silently search against a zero vector. The ordinary path raises + // instead (FunctionArrayDistance::_validate_no_nulls: "Second argument ... + // cannot have null"), and the rewrite must not replace that error with rows + // for a different input, so any null element declines the rewrite. Both + // supported metrics share this. + if (((ArrayLiteral) right).getValue().stream().anyMatch(NullLiteral.class::isInstance)) { + return null; + } + SlotReference vectorSlot = (SlotReference) left; + if (!vectorSlot.getOriginalColumn().isPresent() || !vectorSlot.getOriginalTable().isPresent()) { + return null; + } + String annColumnName = vectorSlot.getOriginalColumn().get().getName(); + + // Validate the PK-vector index and metric on the relation's bound Paimon + // table: a branch table carries its own index options and can differ from + // the main table (see boundPaimonTable). + PaimonExternalTable paimonTable = (PaimonExternalTable) scan.getTable(); + CoreOptions coreOptions; + try { + Table basePaimonTable = boundPaimonTable(paimonTable, scan); + coreOptions = CoreOptions.fromMap(basePaimonTable.options()); + } catch (Throwable t) { + return null; + } + if (!coreOptions.primaryKeyVectorIndexEnabled()) { + return null; + } + boolean columnIndexed = coreOptions.primaryKeyVectorIndexColumns().stream() + .anyMatch(c -> c.equalsIgnoreCase(annColumnName)); + if (!columnIndexed) { + return null; + } + // v1 supports only l2 / inner_product; the function must match the index metric. + String metric = coreOptions.primaryKeyVectorDistanceMetric(annColumnName); + String expectedMetric = l2Dist ? "l2" : "inner_product"; + if (!expectedMetric.equals(metric)) { + return null; + } + + // A data predicate additionally requires the SDK's "physical row read" + // table mode: the predicate itself converts fine, but the pinned + // paimon-rust PkVectorSearchParams::resolve rejects every DATA predicate + // unless deletion vectors are enabled and merge-on-read is disabled — + // so the failure lands AFTER this one-way rewrite replaced the distance + // expression, turning a query that the ordinary path answers into a + // user-facing error. Partition-only conjuncts are enforced by the search + // planning itself and stay fine, so decline only on a non-partition one. + // + // The synthetic distance slot is named after a legal paimon user column + // name (__paimon_search_score_to_dis, see the BE reader). A table that + // actually HAS a column of that name would collide with the synthetic + // slot's block-column matching; decline and run the ordinary path. + // + // A complex-typed column (STRUCT/MAP/nested ARRAY) must ride as a whole + // column: nested pruning rewrites a slot whose fields are accessed — + // Project(struct_element(payload, 'b')) materializes the scan tuple as + // ROW — but the vector reader projects only root names and returns + // the full Arrow value, and the struct SerDe then binds children by + // ORDINAL: the wrong child comes back. Filter conjuncts cannot reach + // this (the whitelist admits only scalar columns); the order-key alias + // is replaced by the synthetic slot and cannot reach it either; so only + // a Project output expression over a complex column has to decline. + // + // A NoneMovableFunction in the projection (assert_true) must keep + // evaluating below the TopN — see projectContainsNonMovableExpression. + // + // A historically evolved schema must stay off the vector reader: its + // materialization decodes projected values through Arrow safe casts, + // which can silently NULL out-of-range values (a TIMESTAMP(3) widened + // to TIMESTAMP(9) reading a year-2300 value) where the ordinary path + // preserves them — see PaimonRustReaderCapabilities.hasRustIncompatibleSchemaHistory. + try { + Table table = boundPaimonTable(paimonTable, scan); + if ((optionalFilter.isPresent() + && filterHasDataConjunct(optionalFilter.get(), table) + && !PaimonScanNode.vectorSearchSupportsDataPredicate(table)) + || table.rowType().getFieldNames().stream().anyMatch( + name -> name.equalsIgnoreCase(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)) + || !complexColumnsRideAsWholeColumns(project, orderKeyAlias) + || projectContainsNonMovableExpression(project, orderKeyAlias) + || (table instanceof FileStoreTable + && PaimonVectorSearch.hasRustIncompatibleSchemaHistory( + (FileStoreTable) table))) { + return null; + } + } catch (Throwable t) { + return null; + } + + // Pre-flight the projection-dependent bounds the vector split would hit at + // the scan node: a PK-vector split has no JNI fallback (it would return an + // empty distance column), so a query whose projected columns would set + // projectedVariant -- or an ORC table with TIMESTAMP_LTZ in its schema, the + // orc-ltz-shape bound -- must not be rewritten at all. Without the rewrite + // those queries run through the ordinary paths, where they are supported. + try { + if (projectedColumnsContainVariant(project, optionalFilter) + || orcSchemaContainsTimestampLtz(boundPaimonTable(paimonTable, scan))) { + return null; + } + } catch (Throwable t) { + return null; + } + + // The thrift-level enum the AnnTopNInfo will carry. expectedMetric is kept as a + // string for the comparison above (coreOptions returns a string), but the + // AnnTopNInfo / wire contract uses TVectorMetric, so map once here. DOT_PRODUCT + // is the thrift enum that paimon-rust's "inner_product" score transform maps to + // (see the BE reader's parse_score_transform). + TVectorMetric expectedMetricEnum = l2Dist ? TVectorMetric.L2 : TVectorMetric.DOT_PRODUCT; + + // Pre-flight the rust eligibility the scan node will gate on. The rewrite is + // one-way: a PK-vector split has no JNI fallback (the JNI reader ignores the + // vector payload and would return a non-ANN scan with an empty distance + // column), so the scan node can only fail loudly — a table whose gate would + // fail (credential provider shape, REST-token IO, query-auth, merge-option + // matrix, deletion-vector merge-on-read, scan-mode, storage scheme or HDFS + // credentials, no rust-capable backend) must not be rewritten at all. Without + // the rewrite the query still runs, computing the distance in Doris. The + // projection-dependent bounds (ORC LTZ, projected variant) need the final scan + // tuple and stay scan-node-side, and the scan node re-evaluates on the + // relation-option-processed table, so it stays authoritative for t@options + // overrides this raw-table pre-flight cannot see. + try { + Table baseTable = boundPaimonTable(paimonTable, scan); + PaimonExternalCatalog catalog = (PaimonExternalCatalog) paimonTable.getCatalog(); + Map storagePropertiesMap = + VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( + catalog.getCatalogProperty().getMetastoreProperties(), + catalog.getCatalogProperty().getStoragePropertiesMap(), + baseTable); + Map backendStorageProperties = + CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); + String tableLocation = baseTable instanceof FileStoreTable + ? ((FileStoreTable) baseTable).location().toString() + : baseTable.options().get("path"); + if (!PaimonVectorSearch.vectorScanEligible(baseTable, tableLocation, + backendStorageProperties, + Env.getCurrentSystemInfo().getAllClusterBackends(true))) { + return null; + } + } catch (Throwable t) { + return null; + } + // NOTE: the FE-side data-predicate interception is intentionally MINIMAL. + // The data predicate is handed in full to the BE via the vector search + // builder's residual filter, where the Rust candidate search stage applies + // it together with deletion vectors. The two planner-side exceptions above + // (unridable conjunct shapes, and the SDK's no-deletion-vector table mode) + // exist because those failures would surface only AFTER the one-way rewrite + // replaced the distance expression. Any other table shape Rust cannot + // handle correctly is Rust's responsibility to reject or answer correctly, + // not the planner's. + + // paimon-rust's vector index operates on f32, so carry the query as floats + // rather than List: it avoids a double→float narrowing on the wire + // and lets PaimonScanNode encode straight into a FLOAT32 TSearchVector. + List vectorLiterals = ((ArrayLiteral) right).getValue(); + float[] annQueryVector = new float[vectorLiterals.size()]; + for (int i = 0; i < vectorLiterals.size(); i++) { + annQueryVector[i] = (float) vectorLiterals.get(i).getDouble(); + } + + // Build the reader-produced distance column, matched by name downstream (BE fills + // any block column named __paimon_search_score_to_dis, bridging it from + // paimon-rust's __paimon_search_score Arrow field). It holds the metric-native + // distance, not the score: the BE reader converts it in place while filling the + // block, so nothing has to be computed here. It is backed by a synthetic hidden + // Column so the SlotDescriptor keeps a non-null column through + // FileQueryScanNode.initSchemaParams; the column is excluded from the + // Parquet/ORC column-position mapping on the scan node. + SlotReference distanceSlot = SlotReference.fromColumn( + StatementScopeIdGenerator.newExprId(), scan.getTable(), + PaimonVectorSearch.getDistanceColumn(), scan.qualified()); + + // Reuse the index-produced value everywhere the same distance expression + // appears, mirroring PushDownVectorTopNIntoOlapScan: + // - orderKeyAlias -> matches the `dis` projection at its Alias root, so that + // item is rebuilt as `dis := ` in one step. + // - orderKeyExpr -> matches the bare `dist_fn(vec, q)` wherever else it + // occurs (filter conjuncts, other projections), so those reuse the index + // value instead of recomputing from the raw vector. + // Both keys map to the same slot, so `dis` stays numerically identical to Doris + // semantics on every path. Note ExpressionUtils.replace rewrites top-down and + // short-circuits on a hit, and Alias.withChildren preserves the ExprId/name, so + // the alias identity the TopN references is never disturbed. + Map replaceMap = Maps.newHashMap(); + replaceMap.put(orderKeyAlias, distanceSlot); + replaceMap.put(orderKeyExpr, distanceSlot); + + Plan plan = scan.withVectorTopN( + distanceSlot, + new AnnTopNInfo(annQueryVector, annColumnName, + topN.getLimit() + topN.getOffset(), expectedMetricEnum)); + if (optionalFilter.isPresent()) { + LogicalFilter filter = optionalFilter.get(); + Set newConjuncts = ExpressionUtils.replace(filter.getConjuncts(), replaceMap); + plan = filter.withConjunctsAndChild(newConjuncts, plan); + } + + List newProjections = ExpressionUtils + .replaceNamedExpressions(project.getProjects(), replaceMap); + LogicalProject newProject = project.withProjectsAndChild(newProjections, plan); + return topN.withChildren(newProject); + } + + /** + * Whether every filter conjunct provably rides into the BE vector search (see + * the call site for the bounded-top-N correctness argument). Conservative by + * design: forms outside this whitelist may well be convertible, but the query + * then simply runs un-rewritten, which is always correct. + */ + private static boolean conjunctsRideIntoVectorSearch(LogicalFilter filter) { + for (Expression conjunct : filter.getConjuncts()) { + if (!conjunctRidesIntoVectorSearch(conjunct)) { + return false; + } + } + return true; + } + + private static boolean conjunctRidesIntoVectorSearch(Expression conjunct) { + if (conjunct instanceof ComparisonPredicate) { + ComparisonPredicate compare = (ComparisonPredicate) conjunct; + return (isVectorFilterColumnSlot(compare.left()) && ridesAsLiteral(compare.right())) + || (ridesAsLiteral(compare.left()) && isVectorFilterColumnSlot(compare.right())); + } + if (conjunct instanceof InPredicate) { + InPredicate in = (InPredicate) conjunct; + // The BE converter encodes the option list under a 16-bit child count + // (see PaimonRustPredicateConverter._convert_in): a bigger list — + // reachable by raising expr_children_limit — is dropped whole and lands + // in the post-fetch residual, re-enabling the bounded-TopK truncation + // this whitelist exists to prevent. Mirror the converter's cap: the + // slot plus the options must fit uint16. + if (in.getOptions().size() + 1 > MAX_IN_PREDICATE_CHILDREN) { + return false; + } + return isVectorFilterColumnSlot(in.getCompareExpr()) + && in.getOptions().stream().allMatch(PushDownVectorTopNIntoPaimonScan::ridesAsLiteral); + } + if (conjunct instanceof IsNull) { + return isVectorFilterColumnSlot(((IsNull) conjunct).child()); + } + if (conjunct instanceof Not && ((Not) conjunct).child() instanceof IsNull) { + return isVectorFilterColumnSlot(((IsNull) ((Not) conjunct).child()).child()); + } + return false; + } + + /** + * A literal the BE paimon-rust predicate converter accepts. A {@link NullLiteral} + * is a Literal and passes the earlier instanceof check, but + * {@code _convert_literal} rejects it ({@code is_null_at(0)}), which rejects the + * WHOLE predicate — an IN option of {@code tag IN (1, 2, NULL)} silently rejects + * the entire IN, leaving it in the post-fetch residual. That reintroduces the + * bounded-TopK truncation the whitelist exists to prevent, so a null literal + * never rides. + */ + private static boolean ridesAsLiteral(Expression expr) { + return expr instanceof Literal && !(expr instanceof NullLiteral); + } + + /** + * Whether the filter carries a conjunct on a non-partition column — the DATA + * half of the predicate, the half the paimon-rust vector search applies per + * candidate row. Every conjunct already passed the riding whitelist, so each + * is a simple comparison / IN / null-check whose slots map to original table + * columns; a conjunct touching only partition columns is enforced by split + * planning and needs no per-row filtering. + */ + private static boolean filterHasDataConjunct(LogicalFilter filter, Table paimonTable) { + Set partitionKeys = new HashSet<>(paimonTable.partitionKeys()); + for (Expression conjunct : filter.getConjuncts()) { + Set slots = conjunct.collect(SlotReference.class::isInstance); + for (SlotReference slot : slots) { + if (slot.getOriginalColumn().isPresent() + && !partitionKeys.contains(slot.getOriginalColumn().get().getName())) { + return true; + } + } + } + return false; + } + + /** + * The relation's bound Paimon table -- branch, time travel, statement fence or + * option overrides included -- instead of the cached main table. A branch is an + * independently configured table: it can carry a different vector-index metric + * (inner-product where main has l2), different deletion-vector options or a + * different schema, and the scan reads the BRANCH's schema through paimon-rust. + * Validating index/metric/eligibility against the main table would accept a + * query whose search then scores and reorders with the branch's metric -- + * wrong results even at full recall. + * + *

Relation options ({@code t@options(...)}) apply over the statement snapshot + * and are resolved through the table's own option-application path; everything + * else rides the relation's bound snapshot captured at binding. + */ + private static Table boundPaimonTable(PaimonExternalTable paimonTable, LogicalFileScan scan) { + if (scan.getScanParams().isPresent() && scan.getScanParams().get().isOptions()) { + return paimonTable.getPaimonTable(scan.getScanParams().get()); + } + return paimonTable.getPaimonTable(scan.getRelationSnapshot()); + } + + /** + * A Project expression whose evaluation must stay BELOW the TopN (a + * NoneMovableFunction, such as {@code assert_true}) still has to raise on rows + * the bounded Top-K would never fetch: on the ordinary path the Project + * evaluates before the TopN and fails for a rejected far row, while + * {@code withVectorTopN} truncates the scan to the near rows first, so the + * assertion never sees them and the query silently succeeds. Any non-movable + * expression outside the order-key alias keeps the ordinary path. This is a + * SELECT expression; the filter residual thread is separate. + */ + private static boolean projectContainsNonMovableExpression( + LogicalProject project, Alias orderKeyAlias) { + for (NamedExpression projection : project.getProjects()) { + if (!(projection instanceof Alias) || projection.equals(orderKeyAlias)) { + continue; + } + if (((Alias) projection).child().anyMatch(e -> e instanceof NoneMovableFunction)) { + return true; + } + } + return false; + } + + /** + * Whether every complex-typed column referenced by the projection rides as a + * whole column. Nested pruning materializes the scan tuple with a REDUCED type + * for a column whose fields are accessed ({@code Project(struct_element(payload, + * 'b'))} prunes {@code payload ROW} to {@code ROW}), but the vector + * reader projects only root names and returns the full Arrow value, whose + * struct SerDe binds children by ordinal — the wrong child comes back. A bare + * slot output keeps the full type — UNLESS it was already pruned (an outer + * field projection can leave an already-pruned bare slot here), whose reduced + * type no longer equals the original column's type; the desc-side backstop + * would fail loudly, so decline instead. A complex slot consumed ONLY by a + * distance function never reaches the scan at all (this rewrite replaces + * every occurrence of the function with the synthetic distance slot, + * order-key alias included), so only any OTHER expression over a complex + * column forces the decline. Filter conjuncts cannot reach this: the shape + * whitelist admits only scalar columns. + */ + private static boolean complexColumnsRideAsWholeColumns(LogicalProject project, + Alias orderKeyAlias) { + for (NamedExpression projection : project.getProjects()) { + if (!(projection instanceof Alias)) { + // A bare passthrough slot output: whole unless already pruned. + if (projection instanceof SlotReference + && !slotRidesWholeColumn((SlotReference) projection)) { + return false; + } + continue; + } + Alias alias = (Alias) projection; + if (alias.equals(orderKeyAlias)) { + continue; + } + if (alias.child() instanceof SlotReference) { + if (!slotRidesWholeColumn((SlotReference) alias.child())) { + return false; + } + continue; + } + Set replacedSlots = new HashSet<>(); + Set distanceCalls = alias.child().collect(e -> + e instanceof L2DistanceApproximate || e instanceof InnerProductApproximate); + for (Expression call : distanceCalls) { + replacedSlots.addAll(call.collect(SlotReference.class::isInstance)); + } + Set slots = alias.child().collect(SlotReference.class::isInstance); + for (SlotReference slot : slots) { + if (!replacedSlots.contains(slot)) { + if (slot.getDataType().isComplexType() || !slotRidesWholeColumn(slot)) { + return false; + } + } + } + } + return true; + } + + /** + * A whole-riding slot keeps the type it was created with from the original + * column ({@code SlotReference.fromColumn}: {@code DataType.fromCatalogType}). + * Nested pruning materializes a REDUCED type ({@code ROW} from + * {@code ROW}) while the original column keeps the full one; only a whole + * slot may ride the vector path, matching the desc-side backstop's + * slot-versus-column comparison. + */ + private static boolean slotRidesWholeColumn(SlotReference slot) { + if (!slot.getDataType().isComplexType() || !slot.getOriginalColumn().isPresent()) { + return true; + } + return slot.getDataType().equals(org.apache.doris.nereids.types.DataType.fromCatalogType( + slot.getOriginalColumn().get().getType())); + } + + private static boolean isVectorFilterColumnSlot(Expression expr) { + if (!(expr instanceof SlotReference)) { + return false; + } + SlotReference slot = (SlotReference) expr; + return slot.getOriginalColumn().isPresent() + && VECTOR_FILTER_SUPPORTED_COLUMN_TYPES.contains( + slot.getOriginalColumn().get().getType()); + } + + /** + * Mirrors the scan node's projectedVariant bound (PaimonScanNode: any slot whose + * Doris type contains a Variant keeps the query off the rust reader, and a + * PK-vector split has no non-rust fallback): any projected or filtered column + * containing Variant declines the rewrite so the query stays on an ordinary + * supported path. + */ + private static boolean projectedColumnsContainVariant( + LogicalProject project, Optional> optionalFilter) { + List expressions = new java.util.ArrayList<>(); + expressions.addAll((project.getProjects())); + optionalFilter.ifPresent(filter -> expressions.addAll(filter.getConjuncts())); + for (Expression expression : expressions) { + Set slots = expression.collect(SlotReference.class::isInstance); + for (SlotReference slotRef : slots) { + if (!slotRef.getOriginalColumn().isPresent()) { + continue; + } + if (PaimonUtil.containsVariant(slotRef.getOriginalColumn().get().getType())) { + return true; + } + } + } + return false; + } + + /** + * Mirrors the scan node's orc-ltz-schema bound: paimon-rust's ORC + * TIMESTAMP_LTZ decoding is not equivalent to Doris semantics yet, and a vector + * split would materialize rows through that decoder with no fallback. Snapshot + * selection is irrelevant here -- this is a physical-file property. + */ + private static boolean orcSchemaContainsTimestampLtz(Table boundTable) { + try { + Table baseTable = boundTable; + if (!(baseTable instanceof FileStoreTable)) { + return false; + } + FileStoreTable fileStoreTable = (FileStoreTable) baseTable; + // Paimon's default file format is ORC; only a PARQUET table is known to + // be free of the orc-ltz shape (an unset key falls to the default). + String fileFormat = CoreOptions.fromMap(fileStoreTable.options()).fileFormatString(); + if (!CoreOptions.FILE_FORMAT_PARQUET.equalsIgnoreCase(String.valueOf(fileFormat))) { + return fileStoreTable.schema().fields().stream() + .anyMatch(field -> containsTimestampLtz(field.type())); + } + return false; + } catch (Throwable t) { + // The table might genuinely be unreadable; the ordinary path will surface + // the real error, and declining the rewrite is the safe answer here. + return true; + } + } + + /** Mirrors PaimonScanNode.containsTimestampLtz (package-private there). */ + private static boolean containsTimestampLtz(DataType type) { + if (type == null) { + return false; + } + if (type.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + return true; + } + if (type instanceof ArrayType) { + return containsTimestampLtz(((ArrayType) type).getElementType()); + } + if (type instanceof MapType) { + MapType mapType = (MapType) type; + return containsTimestampLtz(mapType.getKeyType()) + || containsTimestampLtz(mapType.getValueType()); + } + if (type instanceof MultisetType) { + return containsTimestampLtz(((MultisetType) type).getElementType()); + } + if (type instanceof RowType) { + return ((RowType) type).getFields().stream() + .anyMatch(field -> containsTimestampLtz(field.type())); + } + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AnnTopNInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AnnTopNInfo.java new file mode 100644 index 00000000000000..b4aafc332d1169 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AnnTopNInfo.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.thrift.TVectorMetric; + +import java.util.Arrays; +import java.util.Objects; + +/** + * Primary-key vector (ANN) top-N request pushed down into a file scan (Paimon). + * + *

Groups the query vector, the indexed vector column name, the retrieval + * limit (already folded as {@code user_limit + user_offset}) and the distance + * metric so they travel together through the logical/physical scan plumbing + * instead of as four loose parameters. Immutable. + * + *

{@code metric} is the index's distance metric, validated by the rewrite rule + * to match the distance function ({@code L2} or {@code DOT_PRODUCT}; v1 has no + * cosine). The BE needs it because the reader-produced score column carries a + * metric-dependent transform that it undoes in place, so an unset or wrong metric + * silently corrupts the returned distances. + * + *

{@code queryVector} is {@code float[]} because paimon-rust's vector index + * operates on f32; carrying the query as floats (rather than {@code List}) + * avoids a double→float narrowing on the wire and lets + * {@code PaimonScanNode} encode it straight into a {@link + * org.apache.doris.thrift.TSearchVector} with {@code FLOAT32} element type. + */ +public class AnnTopNInfo { + private final float[] queryVector; + private final String columnName; + private final long limit; + private final TVectorMetric metric; + + /** + * Constructor. {@code metric} must be non-null; see the field assignment below. + * Defensive-copies {@code queryVector} so later mutation by the caller cannot + * corrupt the value that travels through the plan. + */ + public AnnTopNInfo(float[] queryVector, String columnName, long limit, TVectorMetric metric) { + this.queryVector = queryVector == null ? null : queryVector.clone(); + this.columnName = columnName; + this.limit = limit; + // Enforced here rather than at the planning sites downstream. By the time this + // object exists the rewrite has already committed to the distance column, so no + // later stage can safely degrade to a non-vector scan (see + // PaimonScanNode.isVectorSearch); a null metric can only be a programming error + // in a future caller, and failing at construction points straight at it. + this.metric = Objects.requireNonNull(metric, "ANN distance metric can not be null"); + } + + public float[] getQueryVector() { + return queryVector == null ? null : queryVector.clone(); + } + + public String getColumnName() { + return columnName; + } + + public long getLimit() { + return limit; + } + + public TVectorMetric getMetric() { + return metric; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AnnTopNInfo that = (AnnTopNInfo) o; + return limit == that.limit + && Arrays.equals(queryVector, that.queryVector) + && Objects.equals(columnName, that.columnName) + && metric == that.metric; + } + + @Override + public int hashCode() { + int result = Objects.hash(columnName, limit, metric); + result = 31 * result + Arrays.hashCode(queryVector); + return result; + } + + @Override + public String toString() { + return "AnnTopNInfo{column=" + columnName + ", limit=" + limit + + ", metric=" + metric + + ", vectorDim=" + (queryVector == null ? 0 : queryVector.length) + "}"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index 12dfffea883152..eaf8defa771b3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.AnnTopNInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -70,6 +71,10 @@ public class LogicalFileScan extends LogicalCatalogRelation implements SupportPr protected final Optional> cachedOutputs; protected final Optional> relationSchema; protected final Optional relationSnapshot; + // Used for primary-key vector (ANN) top-N push down into the Paimon scan. + // Carries the query vector, the indexed vector column name, and the retrieval + // limit (already user_limit + user_offset) down to PaimonScanNode. + protected final Optional annTopN; /** * Constructor for LogicalFileScan. @@ -122,7 +127,7 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali Optional> cachedSlots, Optional> relationSchema) { this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedSlots, relationSchema, - MvccUtil.getSnapshotFromContext(table)); + MvccUtil.getSnapshotFromContext(table), Optional.empty()); } protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, @@ -132,6 +137,18 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali Optional groupExpression, Optional logicalProperties, Optional> cachedSlots, Optional> relationSchema, Optional relationSnapshot) { + this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, + scanParams, groupExpression, logicalProperties, cachedSlots, relationSchema, + relationSnapshot, Optional.empty()); + } + + protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + SelectedPartitions selectedPartitions, Collection operativeSlots, + List virtualColumns, Optional tableSample, + Optional tableSnapshot, Optional scanParams, + Optional groupExpression, Optional logicalProperties, + Optional> cachedSlots, Optional> relationSchema, + Optional relationSnapshot, Optional annTopN) { super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, operativeSlots, virtualColumns, groupExpression, logicalProperties); this.selectedPartitions = selectedPartitions; @@ -141,6 +158,7 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali this.cachedOutputs = cachedSlots; this.relationSchema = relationSchema; this.relationSnapshot = relationSnapshot; + this.annTopN = annTopN; } private static SelectedPartitions initialSelectedPartitions( @@ -228,7 +246,7 @@ public LogicalFileScan withGroupExpression(Optional groupExpres return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, Optional.of(getLogicalProperties()), - cachedOutputs, relationSchema, relationSnapshot); + cachedOutputs, relationSchema, relationSnapshot, annTopN); } @Override @@ -237,14 +255,14 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedOutputs, - relationSchema, relationSnapshot); + relationSchema, relationSnapshot, annTopN); } public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, Optional.empty(), Optional.of(getLogicalProperties()), - cachedOutputs, relationSchema, relationSnapshot); + cachedOutputs, relationSchema, relationSnapshot, annTopN); } @Override @@ -252,7 +270,7 @@ public LogicalFileScan withRelationId(RelationId relationId) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, Optional.empty(), Optional.empty(), cachedOutputs, - relationSchema, relationSnapshot); + relationSchema, relationSnapshot, annTopN); } @Override @@ -262,7 +280,8 @@ public R accept(PlanVisitor visitor, C context) { @Override public boolean equals(Object o) { - return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions); + return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions) + && Objects.equals(annTopN, ((LogicalFileScan) o).annTopN); } @Override @@ -275,7 +294,8 @@ protected boolean hasSameScanState(LogicalCatalogRelation other) { && Objects.equals(tableSample, that.tableSample) && hasSameSnapshot(tableSnapshot, that.tableSnapshot) && hasSameScanParams(scanParams, that.scanParams) - && hasSameResolvedSnapshot(relationSnapshot, that.relationSnapshot); + && hasSameResolvedSnapshot(relationSnapshot, that.relationSnapshot) + && Objects.equals(annTopN, that.annTopN); } private static boolean hasSameResolvedSnapshot( @@ -452,18 +472,71 @@ public LogicalFileScan withOperativeSlots(Collection operativeSlots) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, Optional.of(getLogicalProperties()), - cachedOutputs, relationSchema, relationSnapshot); + cachedOutputs, relationSchema, relationSnapshot, annTopN); } public LogicalFileScan withCachedOutput(List cachedOutputs) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot, scanParams, groupExpression, Optional.empty(), Optional.of(cachedOutputs), - relationSchema, relationSnapshot); + relationSchema, relationSnapshot, annTopN); } @Override public List getOperativeSlots() { return operativeSlots; } + + public Optional getAnnTopN() { + return annTopN; + } + + /** + * Push primary-key vector (ANN) top-N info into the Paimon scan. Appends a + * reader-produced {@code __paimon_search_score_to_dis} virtual column to the scan + * output and records the query vector / indexed column / retrieval limit. + * + *

The new output is derived from the OLD output plus the distance slot, rather + * than by letting {@code computeLogicalProperties()} recompute it. That is not an + * optimization -- it is required for correctness. Passing {@code Optional.empty()} + * for logicalProperties routes the output through + * {@link LogicalCatalogRelation#computeOutput()}, which mints a FRESH ExprId for + * every base column ({@code exprIdGenerator.getNextId()}, no per-column reuse + * cache). The caller (PushDownVectorTopNIntoPaimonScan) reparents the EXISTING + * projections onto this new scan, so those projections would then reference slots + * the scan no longer produces -- e.g. project reads {@code id#10005} while the scan + * outputs {@code id#10010}. Verified by the exprId-stability assertion in + * PushDownVectorTopNIntoPaimonScanTest. + * + *

Same shape as {@code LogicalOlapScan.appendVirtualColumnsAndTopN}, for the same + * reason. Because the output is supplied explicitly, {@code cachedOutputs} can be + * forwarded too: it only feeds {@code computeOutput()}, which is now unreachable, + * and dropping it would lose an upstream rewrite. + * + * @param scoreVirtualColumn the {@code __paimon_search_score_to_dis} output column + * @param annTopN the query vector, indexed column name, and retrieval limit + */ + public LogicalFileScan withVectorTopN( + NamedExpression scoreVirtualColumn, + AnnTopNInfo annTopN) { + List mergedVirtualColumns = ImmutableList.builder() + .addAll(virtualColumns) + .add(scoreVirtualColumn) + .build(); + LogicalProperties oldProperties = getLogicalProperties(); + List newOutput = ImmutableList.builder() + .addAll(oldProperties.getOutput()) + .add(scoreVirtualColumn.toSlot()) + .build(); + // The asterisk output is deliberately NOT extended: `select *` must not surface + // an internal distance column. Forwarded as a supplier rather than a value so + // that this rewrite does not force a computation it never reads -- the asterisk + // output only matters during binding, long before this rule runs. + LogicalProperties newProperties = new LogicalProperties( + () -> newOutput, oldProperties::getAsteriskOutput, this::computeDataTrait); + return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, + selectedPartitions, operativeSlots, mergedVirtualColumns, tableSample, tableSnapshot, + scanParams, groupExpression, Optional.of(newProperties), cachedOutputs, + relationSchema, relationSnapshot, Optional.of(annTopN)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java index 2696fe6e52a197..bf60d5e71702f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.AnnTopNInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -50,6 +51,8 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { protected final Optional tableSnapshot; protected final Optional scanParams; protected final Optional relationSnapshot; + // Used for primary-key vector (ANN) top-N push down into the Paimon scan. + private final Optional annTopN; /** * Constructor for PhysicalFileScan. @@ -74,7 +77,20 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi Optional scanParams, Optional relationSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, - relationSnapshot); + relationSnapshot, Optional.empty()); + } + + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot, + Collection operativeSlots, + Optional scanParams, Optional relationSnapshot, + Optional annTopN) { + this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, + logicalProperties, selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, + relationSnapshot, annTopN); } /** @@ -100,7 +116,19 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi Optional relationSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams, relationSnapshot); + operativeSlots, scanParams, relationSnapshot, Optional.empty()); + } + + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot, + Collection operativeSlots, Optional scanParams, + Optional relationSnapshot, Optional annTopN) { + this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, + logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, + operativeSlots, scanParams, relationSnapshot, annTopN); } /** @@ -124,6 +152,19 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li Optional tableSnapshot, Collection operativeSlots, Optional scanParams, Optional relationSnapshot) { + this(id, type, table, qualifier, distributionSpec, groupExpression, logicalProperties, + selectedPartitions, tableSample, tableSnapshot, operativeSlots, scanParams, + relationSnapshot, Optional.empty()); + } + + protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot, + Collection operativeSlots, + Optional scanParams, Optional relationSnapshot, + Optional annTopN) { super(id, type, table, qualifier, groupExpression, logicalProperties, operativeSlots); this.distributionSpec = distributionSpec; this.selectedPartitions = selectedPartitions; @@ -131,6 +172,7 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li this.tableSnapshot = tableSnapshot; this.scanParams = scanParams; this.relationSnapshot = relationSnapshot; + this.annTopN = annTopN; } protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, @@ -151,6 +193,18 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li Optional tableSample, Optional tableSnapshot, Collection operativeSlots, Optional scanParams, Optional relationSnapshot) { + this(id, type, table, qualifier, distributionSpec, groupExpression, logicalProperties, + physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, + operativeSlots, scanParams, relationSnapshot, Optional.empty()); + } + + protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot, + Collection operativeSlots, Optional scanParams, + Optional relationSnapshot, Optional annTopN) { super(id, type, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics, operativeSlots); this.distributionSpec = distributionSpec; @@ -159,6 +213,7 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li this.tableSnapshot = tableSnapshot; this.scanParams = scanParams; this.relationSnapshot = relationSnapshot; + this.annTopN = annTopN; } public DistributionSpec getDistributionSpec() { @@ -189,6 +244,10 @@ public Optional getRelationSnapshot() { return relationSnapshot; } + public Optional getAnnTopN() { + return annTopN; + } + @Override public String toString() { String rfV2 = ""; @@ -214,7 +273,7 @@ public R accept(PlanVisitor visitor, C context) { public PhysicalFileScan withGroupExpression(Optional groupExpression) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams, relationSnapshot); + operativeSlots, scanParams, relationSnapshot, annTopN); } @Override @@ -222,7 +281,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams, relationSnapshot); + operativeSlots, scanParams, relationSnapshot, annTopN); } @Override @@ -236,7 +295,7 @@ public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physic return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, - operativeSlots, scanParams, relationSnapshot); + operativeSlots, scanParams, relationSnapshot, annTopN); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java index 4c0befccf97005..ade669b35c030b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java @@ -42,7 +42,7 @@ public PhysicalLazyMaterializeFileScan(PhysicalFileScan scan, SlotReference rowI Optional.empty(), null, null, scan.getStats(), scan.selectedPartitions, scan.getTableSample(), scan.getTableSnapshot(), scan.getOperativeSlots(), - scan.getScanParams(), scan.getRelationSnapshot()); + scan.getScanParams(), scan.getRelationSnapshot(), scan.getAnnTopN()); this.scan = scan; this.rowId = rowId; this.lazySlots = lazySlots; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index f6eaf55e5f13c5..995b727e61d65b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -3116,6 +3116,7 @@ public Map getForceEagerAggHintMap() { "Use paimon-rust for non-native Paimon reads, higher priority than paimon-cpp"}) private boolean enablePaimonRustReader = false; + @VariableMgr.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE, fuzzy = true, description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown optimization for external table"}) diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java index adb4a57d448385..885e240503ce93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java @@ -147,6 +147,12 @@ public class Backend implements Writable { private long beMemory = 0; @SerializedName("supportsPaimonRustReader") private volatile boolean supportsPaimonRustReader; + // The PK-vector (ANN) search protocol bit, negotiated separately from the + // batch rust reader: an older rust-capable BE reports only the batch bit and + // rejects a vector payload ("missing paimon_split"), so the one-way vector + // rewrite must require this bit specifically. + @SerializedName("supportsPaimonRustVectorReader") + private volatile boolean supportsPaimonRustVectorReader; // from config::pipeline_executor_size , default equal cpuCores @SerializedName("pipelineExecutorSize") private int pipelineExecutorSize = 1; @@ -263,6 +269,10 @@ public boolean isPaimonRustReaderSupported() { return supportsPaimonRustReader; } + public boolean isPaimonRustVectorReaderSupported() { + return supportsPaimonRustVectorReader; + } + public int getBePort() { return bePort; } @@ -882,6 +892,10 @@ public boolean handleHbResponse(BackendHbResponse hbResponse, boolean isReplay) isChanged = true; supportsPaimonRustReader = hbResponse.isPaimonRustReaderSupported(); } + if (supportsPaimonRustVectorReader != hbResponse.isPaimonRustVectorReaderSupported()) { + isChanged = true; + supportsPaimonRustVectorReader = hbResponse.isPaimonRustVectorReaderSupported(); + } if (!this.version.equals(hbResponse.getVersion())) { isChanged = true; this.version = hbResponse.getVersion(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java index 55fb0c9f456a74..42b19395c3efe6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java @@ -53,6 +53,10 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable { private long beMemory = 0; @SerializedName("supportsPaimonRustReader") private boolean supportsPaimonRustReader; + // The PK-vector (ANN) search protocol bit, negotiated separately: an older + // rust-capable BE reports the batch bit but would reject a vector payload. + @SerializedName("supportsPaimonRustVectorReader") + private boolean supportsPaimonRustVectorReader; public BackendHbResponse() { super(HeartbeatResponse.Type.BACKEND); @@ -162,6 +166,14 @@ public void setPaimonRustReaderSupported(boolean supported) { this.supportsPaimonRustReader = supported; } + public boolean isPaimonRustVectorReaderSupported() { + return supportsPaimonRustVectorReader; + } + + public void setPaimonRustVectorReaderSupported(boolean supported) { + this.supportsPaimonRustVectorReader = supported; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java index 71e053995c1f07..922b22eae01828 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java @@ -379,6 +379,9 @@ private HeartbeatResponse pingOnce() { fragmentNum, lastFragmentUpdateTime, isShutDown, arrowFlightSqlPort, beMemory); response.setPaimonRustReaderSupported(tBackendInfo.isSetSupportsPaimonRustReader() && tBackendInfo.isSupportsPaimonRustReader()); + response.setPaimonRustVectorReaderSupported( + tBackendInfo.isSetSupportsPaimonRustVectorReader() + && tBackendInfo.isSupportsPaimonRustVectorReader()); return response; } else { return new BackendHbResponse(backendId, backend.getHost(), backend.getLastUpdateMs(), diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 4608013b5fbf4d..dcdeff4024dd8b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -54,8 +54,6 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -1829,16 +1827,15 @@ public void testAdvancingLatestFenceKeepsOnlyNewestProjectionOfTableGeneration() } @Test - public void testSnapshotEstimateSupportsPrivilegedTableWrapper() throws Exception { - FileStoreTable table = newPartitionedTable("privileged_estimate", Collections.emptyMap()); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - table, Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "tbl")); + public void testSnapshotEstimateSupportsDelegateTableWrapper() throws Exception { + FileStoreTable table = newPartitionedTable("delegate_estimate", Collections.emptyMap()); + FileStoreTable delegate = new PlainDelegateFileStoreTable(table); NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( mapping, 1L, table.schema().id(), 1L); PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( PaimonPartitionInfo.EMPTY, - new PaimonSnapshot(1L, table.schema().id(), privileged)); + new PaimonSnapshot(1L, table.schema().id(), delegate)); value.prepareForCachePublication(key); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java index aa1d6415adedd8..0e022941bc02e3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java @@ -22,11 +22,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AppendOnlyFileStoreTable; import org.apache.paimon.table.CatalogEnvironment; @@ -130,10 +127,12 @@ void testRuntimeCapNormalizesEveryPlanningLeafIndependently() { FallbackReadFileStoreTable emptyThenExplicit = (FallbackReadFileStoreTable) PaimonReaderOptions.runtimeSafeTable( new FallbackReadFileStoreTable(empty, explicit, true), 512); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - new FallbackReadFileStoreTable(explicit, empty, true), - Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "table")); - Table normalizedDelegate = PaimonReaderOptions.runtimeSafeTable(privileged, 512); + // Paimon 2.1 no longer produces non-fallback delegates, so an unknown one is rejected + // instead of being peeled with a single cap broadcast to every hidden branch. + Table unknownDelegate = new PlainDelegateFileStoreTable( + new FallbackReadFileStoreTable(explicit, empty, true)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> PaimonReaderOptions.runtimeSafeTable(unknownDelegate, 512)); Assertions.assertEquals("256", safeEmpty.options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); @@ -145,9 +144,6 @@ void testRuntimeCapNormalizesEveryPlanningLeafIndependently() { .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); Assertions.assertEquals("1", emptyThenExplicit.other().options() .get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); - Assertions.assertInstanceOf(FallbackReadFileStoreTable.class, normalizedDelegate); - Assertions.assertEquals("256", ((FallbackReadFileStoreTable) normalizedDelegate) - .other().options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key())); } @Test @@ -186,11 +182,9 @@ void testSystemSourceKeepsFallbackAsOutermostPlanningDecorator() { FileStoreTable main = newFileStoreTable("main", Collections.emptyMap()); FileStoreTable fallback = newFileStoreTable("fallback", Collections.emptyMap()); FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - pair, Mockito.mock(PrivilegeChecker.class), - Identifier.create("db", "privileged_fallback")); + FileStoreTable delegate = new PlainDelegateFileStoreTable(pair); - Assertions.assertSame(pair, PaimonTableDecorators.unwrapToFallbackOrBase(privileged)); + Assertions.assertSame(pair, PaimonTableDecorators.unwrapToFallbackOrBase(delegate)); } @Test @@ -282,18 +276,15 @@ void testBackendManifestCapUsesExecutionCeilingNotSmallestBranch() { } @Test - void testRejectUnsafeFallbackHiddenByPrivilegeDelegate() { - FileStoreTable main = newFileStoreTable("privileged_main", Collections.emptyMap()); + void testRejectUnsafeFallbackHiddenByDelegate() { + FileStoreTable main = newFileStoreTable("delegate_main", Collections.emptyMap()); FileStoreTable fallback = newFileStoreTable( - "privileged_fallback", ImmutableMap.of("scan.manifest.parallelism", "0")); + "delegate_fallback", ImmutableMap.of("scan.manifest.parallelism", "0")); FileStoreTable fallbackReadTable = new FallbackReadFileStoreTable(main, fallback, true); - FileStoreTable privilegedTable = PrivilegedFileStoreTable.wrap( - fallbackReadTable, - Mockito.mock(PrivilegeChecker.class), - Identifier.create("db", "table")); + FileStoreTable delegateTable = new PlainDelegateFileStoreTable(fallbackReadTable); Assertions.assertThrows(IllegalArgumentException.class, - () -> PaimonReaderOptions.validateEffectiveTable(privilegedTable)); + () -> PaimonReaderOptions.validateEffectiveTable(delegateTable)); } private FileStoreTable newFileStoreTable(String name, Map options) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonRowCountTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonRowCountTest.java index bba3e27f3ac5c5..6499347e7ea2db 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonRowCountTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonRowCountTest.java @@ -22,13 +22,10 @@ import com.google.common.collect.ImmutableMap; import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; @@ -124,23 +121,6 @@ public void testUnsupportedTablesDoNotPlan() throws Exception { rowCount(new FallbackReadFileStoreTable(main, other, true))); } - @Test - public void testSelectOnlyPrivilegeWrapper() throws Exception { - FileStoreTable table = newTable(false); - snapshot(table, 1, 10L); - PrivilegeChecker checker = Mockito.mock(PrivilegeChecker.class); - Identifier identifier = Identifier.create("db", "tbl"); - Mockito.doThrow(new IllegalStateException("INSERT is not granted")) - .when(checker).assertCanInsert(identifier); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap(table, checker, identifier); - Assert.assertEquals(10L, rowCount(privileged)); - Mockito.verify(checker).assertCanSelect(identifier); - Mockito.verify(checker, Mockito.never()).assertCanInsert(identifier); - Mockito.doThrow(new IllegalStateException("SELECT is not granted")) - .when(checker).assertCanSelect(identifier); - Assert.assertThrows(IllegalStateException.class, () -> rowCount(privileged)); - } - @Test public void testCatalogQueryAuthorizationWithoutPlanning() throws Exception { FileStoreTable table = Mockito.spy(newTable(false).copy(ImmutableMap.of("query-auth.enabled", "true"))); @@ -175,15 +155,15 @@ private FileStoreTable newTable(boolean primaryKey, Map options) schema.primaryKey("id").option("bucket", "1"); } options.forEach(schema::option); - new SchemaManager(fileIO, path).createTable(schema.build()); + new FileSystemSchemaManager(fileIO, path).createTable(schema.build()); return FileStoreTableFactory.create(fileIO, path); } private void snapshot(FileStoreTable table, long id, long count) throws Exception { // No manifest files exist: accidentally returning to split planning must fail. Snapshot snapshot = new Snapshot(id, 0L, "unused-base", null, "unused-delta", null, - null, null, null, "test", id, Snapshot.CommitKind.APPEND, id * 1000, - count, count, null, null, null, Collections.emptyMap(), null); + null, null, null, "test", null, id, Snapshot.CommitKind.APPEND, id * 1000, + count, count, null, null, null, Collections.emptyMap(), null, null); table.fileIO().mkdirs(table.snapshotManager().snapshotPath(id).getParent()); table.fileIO().overwriteFileUtf8(table.snapshotManager().snapshotPath(id), snapshot.toJson()); table.snapshotManager().commitLatestHint(id); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 38b0169741ed03..8c40b4888962c1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -52,6 +52,7 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; import org.apache.paimon.types.VarCharType; +import org.apache.paimon.types.VectorType; import org.apache.thrift.TDeserializer; import org.apache.thrift.TSerializer; import org.junit.Assert; @@ -666,4 +667,78 @@ private static String repeatedCharacter(char character, int count) { Arrays.fill(characters, character); return new String(characters); } + + // ------------------------------------------------------------------ VECTOR + + /** + * A Paimon VECTOR column must map to ARRAY<element>, not UNSUPPORTED. This is what + * makes the column usable at all: the ANN distance functions are declared over + * ARRAY<FLOAT> (L2DistanceApproximate.SIGNATURES), so an UNSUPPORTED mapping means + * l2_distance_approximate(embedding, [...]) never type-checks and the ANN rewrite + * (PushDownVectorTopNIntoPaimonScan) is never reached -- the whole PK-vector search + * path is unreachable on a real table. + */ + @Test + public void testVectorTypeMapsToArray() { + DataField vectorField = new DataField(1, "embedding", + new VectorType(2048, DataTypes.FLOAT())); + Type type = PaimonUtil.paimonTypeToDorisType(vectorField.type(), true, true); + + Assert.assertTrue("VECTOR must not map to UNSUPPORTED: " + type, type.isArrayType()); + Assert.assertEquals(Type.FLOAT, ((ArrayType) type).getItemType()); + } + + /** + * The element type is carried through rather than hardcoded to FLOAT. VectorType + * permits any of BOOLEAN/TINYINT/SMALLINT/INTEGER/BIGINT/FLOAT/DOUBLE + * (VectorType.isValidElementType), and only FLOAT is exercised by ANN today. + */ + @Test + public void testVectorTypeCarriesElementType() { + Type doubleVec = PaimonUtil.paimonTypeToDorisType( + new VectorType(8, DataTypes.DOUBLE()), true, true); + Assert.assertEquals(Type.DOUBLE, ((ArrayType) doubleVec).getItemType()); + + Type intVec = PaimonUtil.paimonTypeToDorisType( + new VectorType(4, DataTypes.INT()), true, true); + Assert.assertEquals(Type.INT, ((ArrayType) intVec).getItemType()); + } + + /** + * The thrift schema handed to BE must describe the element too. VECTOR shares the + * ARRAY branch of getSchemaInfo for exactly this reason: the default branch only sets + * the primitive type, which would declare TPrimitiveType.ARRAY with no item field. + */ + @Test + public void testVectorTypeSchemaInfoCarriesItemField() { + TField field = PaimonUtil.getSchemaInfo(new VectorType(2048, DataTypes.FLOAT()), true, true); + + Assert.assertEquals(TPrimitiveType.ARRAY, field.getType().getType()); + Assert.assertTrue("VECTOR schema must carry a nested item field, else BE sees an " + + "ARRAY with no element description", field.isSetNestedField()); + Assert.assertEquals(TPrimitiveType.FLOAT, field.getNestedField().getArrayField() + .getItemField().getFieldPtr().getType().getType()); + } + + /** + * The array child column of a VECTOR must receive the Paimon element field id. Guards a + * real hazard rather than a hypothetical one: updatePaimonColumnUniqueId indexes + * children.get(0) directly and only null-checks the list, so recursing into a VECTOR is + * only safe because Column.createChildrenColumn builds a child for any isArrayType(). + */ + @Test + public void testVectorTypeChildColumnGetsElementFieldId() { + DataField elementField = new DataField(7, "element", DataTypes.FLOAT()); + VectorType paimonVectorType = new VectorType(true, 2048, elementField.type()); + DataField vectorField = new DataField(3, "embedding", paimonVectorType); + + Column column = new Column("embedding", + PaimonUtil.paimonTypeToDorisType(vectorField.type(), true, true), true); + PaimonUtil.updatePaimonColumnUniqueId(column, vectorField); + + Assert.assertEquals(3, column.getUniqueId()); + Assert.assertNotNull("a VECTOR-mapped ARRAY must have its array child column, otherwise " + + "updatePaimonColumnUniqueId would throw on children.get(0)", column.getChildren()); + Assert.assertEquals(1, column.getChildren().size()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PlainDelegateFileStoreTable.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PlainDelegateFileStoreTable.java new file mode 100644 index 00000000000000..b8cb446953338c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PlainDelegateFileStoreTable.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.DelegatedFileStoreTable; +import org.apache.paimon.table.FileStoreTable; + +import java.util.Map; + +/** + * Minimal non-fallback delegate used to exercise the delegate-peeling paths that the removed + * Paimon privilege wrapper used to cover. + */ +final class PlainDelegateFileStoreTable extends DelegatedFileStoreTable { + + PlainDelegateFileStoreTable(FileStoreTable wrapped) { + super(wrapped); + } + + @Override + public FileStoreTable copy(Map dynamicOptions) { + return new PlainDelegateFileStoreTable(wrapped().copy(dynamicOptions)); + } + + @Override + public FileStoreTable copy(TableSchema newTableSchema) { + return new PlainDelegateFileStoreTable(wrapped().copy(newTableSchema)); + } + + @Override + public FileStoreTable copyWithoutTimeTravel(Map dynamicOptions) { + return new PlainDelegateFileStoreTable(wrapped().copyWithoutTimeTravel(dynamicOptions)); + } + + @Override + public FileStoreTable copyWithLatestSchema() { + return new PlainDelegateFileStoreTable(wrapped().copyWithLatestSchema()); + } + + @Override + public FileStoreTable switchToBranch(String branchName) { + return new PlainDelegateFileStoreTable(wrapped().switchToBranch(branchName)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java index 4e44aa67809e77..11be03fdd197fd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderCompatibilityTest.java @@ -30,9 +30,9 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.sink.BatchTableCommit; @@ -136,9 +136,11 @@ public void testPersistedIntegerToTimestamp() throws Exception { @Test public void testPersistedTimestampToDateBeforeEpoch() throws Exception { - // Java divides toward zero; Arrow takes the preceding calendar date before the epoch. + // Before the epoch the persisted timestamp-to-date conversion takes the + // preceding calendar date (floor, like Arrow); the internal 2.1 BETA build + // truncated toward zero (Java integer division) and returned day 0. assertPersistedEvolution(DataTypes.TIMESTAMP(3), DataTypes.DATE(), Timestamp.fromEpochMillis(-1), - false, row -> row.getInt(1), 0); + false, row -> row.getInt(1), -1); } @Test @@ -461,7 +463,7 @@ private FileStoreTable table(Schema.Builder schema, Map options) .option("file.format", "parquet").option("deletion-vectors.enabled", "false") .option("read.batch-size", "16").option("scan.manifest.parallelism", "1"); options.forEach(schema::option); - new SchemaManager(fileIO, path).createTable(schema.build()); + new FileSystemSchemaManager(fileIO, path).createTable(schema.build()); return FileStoreTableFactory.create(fileIO, path); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java index 1ed35e94007985..4c436852bdfee0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonRustReaderLegacySchemaTest.java @@ -25,9 +25,9 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.sink.BatchTableCommit; @@ -56,7 +56,7 @@ public void testPersistedLegacySchemaFallsBackBeforeAndAfterEvolution() throws E java.nio.file.Path directory = temporaryFolder.newFolder().toPath(); Path location = new Path(directory.toUri()); LocalFileIO fileIO = LocalFileIO.create(); - new SchemaManager(fileIO, location).createTable(Schema.newBuilder() + new FileSystemSchemaManager(fileIO, location).createTable(Schema.newBuilder() .column("id", DataTypes.INT()).option("bucket", "-1") .option("file.format", "parquet").build()); FileStoreTable table = FileStoreTableFactory.create(fileIO, location); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 2508f3c7bbee6e..ea210bb4871d4e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -81,6 +81,7 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.rest.RESTTokenFileIO; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.table.AppendOnlyFileStoreTable; @@ -146,8 +147,12 @@ public void saveVariantV2Config() { // Statement-scoped scan task reuse is on by default; the mock's field would otherwise // read false and bypass the cache under test. sv.enableExternalScanTaskReuse = true; - // These routing fixtures bypass scan initialization and use an upgraded eligible BE. - Backend backend = GsonUtils.GSON.fromJson("{\"supportsPaimonRustReader\":true}", Backend.class); + // These routing fixtures bypass scan initialization and use an upgraded eligible + // BE. The vector bit rides along: the vector gates negotiate it independently + // of the batch one, and the vector-split tests below need both. + Backend backend = GsonUtils.GSON.fromJson( + "{\"supportsPaimonRustReader\":true,\"supportsPaimonRustVectorReader\":true}", + Backend.class); new MockUp() { @mockit.Mock public Collection getBackends() { @@ -1174,6 +1179,122 @@ public void testBoundEmptySnapshotDoesNotReadLaterCommit() throws Exception { Mockito.verify(liveTable, Mockito.never()).newReadBuilder(); } + @Test + public void testVectorSearchSplitsFencePinnedEmptyGeneration() throws Exception { + // The vector twin of testBoundEmptySnapshotDoesNotReadLaterCommit: the relation + // bound the pinned-empty generation (no commit existed at binding). The + // ordinary path fences this to empty; without the fence here, the vector + // builder resolves the table's LATEST and exposes a first commit that landed + // after binding, breaking read consistency between the two planning paths. + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + Table liveTable = Mockito.mock(Table.class); + node.setSource(source); + node.setRelationSnapshot(Optional.of(new PaimonMvccSnapshot( + new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(PaimonSnapshot.INVALID_SNAPSHOT_ID, 1L, liveTable))))); + + Assert.assertTrue(node.getVectorSearchSplits().isEmpty()); + Mockito.verify(liveTable, Mockito.never()).newBatchVectorSearchBuilder(); + } + + @Test + public void testVectorSplitFailsLoudOnIncompatibleFileSchema() throws Exception { + // A vector split whose file carries a historically incompatible schema id is a + // broken invariant (the rewrite's hasRustIncompatibleSchemaHistory pre-flight + // must decline such a table): the vector reader would materialize projected + // values through the incompatible Arrow decoder with no JNI fallback, so the + // scan node fails loudly instead of silently corrupting values. + PaimonScanNode node = vectorRoutingNode(); + Mockito.when(vectorProcessedTable.schemaManager().schema(1L)) + .thenReturn(incompatibleHistorySchema()); + PaimonSplit split = new PaimonSplit(createDataSplit("vector_hist.parquet")); + split.setVectorPayloadBytes(new byte[] {1}); + + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + Throwable thrown = null; + try { + invokePrivateMethod(node, "setPaimonParams", + new Class[] {TFileRangeDesc.class, PaimonSplit.class}, rangeDesc, split); + } catch (java.lang.reflect.InvocationTargetException e) { + thrown = e.getCause(); + } + Assert.assertTrue("expected the loud vector-split failure, got " + thrown, + thrown instanceof RuntimeException); + Assert.assertTrue("the failure must name the rust-reader requirement, got " + thrown, + thrown.getMessage().contains("paimon-rust reader")); + } + + @Test + public void testVectorSplitWithMatchingFileSchemaStillEncodes() throws Exception { + // Control: the split's file carries the CURRENT schema id (no history to + // decode differently), so the vector payload is encoded and the split rides + // the rust reader with an empty paimon_split (the DataSplit is already inside + // vector_payload). + PaimonScanNode node = vectorRoutingNode(); + // The current schema id equals the file's schema id (1): the per-file history + // gate never fires. + Mockito.when(vectorProcessedTable.schema()) + .thenReturn(new TableSchema(1, + Collections.singletonList(new DataField(0, "id", new IntType())), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null)); + PaimonSplit split = new PaimonSplit(createDataSplit("vector_ok.parquet")); + split.setVectorPayloadBytes(new byte[] {1}); + + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + invokePrivateMethod(node, "setPaimonParams", + new Class[] {TFileRangeDesc.class, PaimonSplit.class}, rangeDesc, split); + org.apache.doris.thrift.TPaimonFileDesc fileDesc = + rangeDesc.getTableFormatParams().getPaimonParams(); + Assert.assertEquals( + org.apache.doris.thrift.TPaimonReaderType.PAIMON_RUST, fileDesc.getReaderType()); + Assert.assertEquals(1, fileDesc.getVectorPayload().getBytes().length); + Assert.assertTrue(fileDesc.isSetPaimonSplit() && fileDesc.getPaimonSplit().isEmpty()); + } + + private FileStoreTable vectorProcessedTable; + + /** A routing node for vector splits: rust+V2 on, a processed FileStoreTable with a + * current schema id 5 and per-test history stubs, backend/storage properties open. */ + private PaimonScanNode vectorRoutingNode() throws Exception { + SessionVariable vars = new SessionVariable(); + vars.setEnablePaimonRustReader(true); + vars.enableFileScannerV2 = true; + PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0), + new TupleDescriptor(new TupleId(0)), false, vars, ScanContext.EMPTY); + vectorProcessedTable = Mockito.mock(FileStoreTable.class); + // Lenient stubbings: which are consumed varies between the two vector-split + // tests (the control re-stubs schema() to match the file's schema id). + Mockito.lenient().when(vectorProcessedTable.options()).thenReturn(Collections.emptyMap()); + Mockito.lenient().when(vectorProcessedTable.schema()).thenReturn(new TableSchema(5, + Collections.singletonList(new DataField(0, "id", new IntType())), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null)); + Mockito.lenient().when(vectorProcessedTable.schemaManager()) + .thenReturn(Mockito.mock(SchemaManager.class)); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + Mockito.when(source.getExternalTable()).thenReturn(externalTable); + Mockito.when(source.getTableLocation()).thenReturn("s3://warehouse/wh/db.db/t"); + Mockito.when(externalTable.getDbName()).thenReturn("db"); + Mockito.when(externalTable.getName()).thenReturn("t"); + node.setSource(source); + setField(PaimonScanNode.class, node, "processedTable", vectorProcessedTable); + setField(PaimonScanNode.class, node, "backendStorageProperties", + ImmutableMap.of("AWS_ACCESS_KEY", "example-key", "AWS_SECRET_KEY", "example-secret")); + setField(PaimonScanNode.class, node, "storagePropertiesMap", Collections.emptyMap()); + return node; + } + + /** A schema-1 history whose ts TIMESTAMP(3) cannot decode under the current schema. */ + private TableSchema incompatibleHistorySchema() { + return new TableSchema(1, + Collections.singletonList(new DataField(0, "ts", new TimestampType(3))), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null); + } + @Test public void testBoundEmptyDataSnapshotStillPlansMetadataSystemTable() throws Exception { // A live statement scope is required: the metadata-system-table path plans through the @@ -2916,6 +3037,114 @@ public void testRustReaderSelectionRequiresExplicitS3Credentials() throws Except rangeDesc.getTableFormatParams().getPaimonParams().getReaderType()); } } + // --- filterVectorIndexOptions --------------------------------------------------- + // + // CoreOptions.primaryKeyVectorIndexOptions seeds its result with a copy of every + // table option, so what reaches TExternalSearchRequest.paimon_options is the whole table + // property bag unless it is filtered. These cover the three reasons it must be: + // a thrift-fatal null value, redundant table options paimon-rust already merges + // itself, and properties that should not travel to BE at all. + + @Test + public void testFilterVectorIndexOptionsKeepsFieldAndAlgorithmScopedKeys() { + Map options = new HashMap<>(); + options.put("field.embedding.vector-dim", "2048"); + options.put("ivf-flat.metric", "l2"); + options.put("ivf-flat.nlist", "1024"); + options.put("fields.embedding.some-field-option", "v"); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertEquals("l2", filtered.get("ivf-flat.metric")); + Assert.assertEquals("1024", filtered.get("ivf-flat.nlist")); + Assert.assertEquals("v", filtered.get("fields.embedding.some-field-option")); + // "field." (singular) is a different namespace from the "fields." one the filter + // keeps, so the dimension is not forwarded -- paimon-rust reads it off the index. + Assert.assertFalse(filtered.containsKey("field.embedding.vector-dim")); + Assert.assertEquals(3, filtered.size()); + } + + @Test + public void testFilterVectorIndexOptionsDropsNullValues() { + // The actual production failure: these catalog audit properties carry a Java null, + // which HashMap accepts and thrift's writeString does not (NPE in the generated + // TPaimonVectorSearchOptions writer). They are also not algorithm- or field-scoped, + // so they would be dropped anyway -- asserted separately so a future widening of + // the prefix rule cannot silently reintroduce the crash. + Map options = new HashMap<>(); + options.put("owner", null); + options.put("createdBy", null); + options.put("updatedBy", null); + options.put("ivf-flat.metric", "l2"); + options.put("ivf-flat.nprobe", null); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertFalse(filtered.containsValue(null)); + Assert.assertFalse(filtered.containsKey("ivf-flat.nprobe")); + Assert.assertEquals(Collections.singletonMap("ivf-flat.metric", "l2"), filtered); + } + + @Test + public void testFilterVectorIndexOptionsDropsUnrelatedTableProperties() { + Map options = new HashMap<>(); + options.put("path", "viewfs://cluster/db/tbl"); + options.put("security.hadoop.username", "hudi"); + options.put("bucket", "1"); + options.put("primary-key", "id"); + options.put("merge-engine", "partial-update"); + options.put("ivf-flat.metric", "l2"); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertEquals(Collections.singletonMap("ivf-flat.metric", "l2"), filtered); + } + + @Test + public void testFilterVectorIndexOptionsDropsPkVectorDeclarations() { + // fields..pk-vector.* is FE's own index declaration, not a search parameter; + // primaryKeyVectorAlgorithmOptions excludes it from its collection too. + Map options = new HashMap<>(); + options.put("fields.embedding.pk-vector.index.type", "ivf-flat"); + options.put("fields.embedding.pk-vector.distance.metric", "l2"); + options.put("fields.embedding.pk-vector.index.options", "{\"nlist\":\"1024\"}"); + options.put("fields.embedding.refine-factor", "2"); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertEquals(Collections.singletonMap("fields.embedding.refine-factor", "2"), filtered); + } + + @Test + public void testFilterVectorIndexOptionsWithNullAlgorithmKeepsFieldScopedOnly() { + // A null index type must not be turned into a "null." prefix; keep the + // field-scoped half rather than guessing. + Map options = new HashMap<>(); + options.put("fields.embedding.refine-factor", "2"); + options.put("ivf-flat.metric", "l2"); + options.put("owner", null); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", null); + + Assert.assertEquals(Collections.singletonMap("fields.embedding.refine-factor", "2"), filtered); + } + + @Test + public void testFilterVectorIndexOptionsIsScopedToTheQueriedColumn() { + Map options = new HashMap<>(); + options.put("fields.embedding.refine-factor", "2"); + options.put("fields.other_vec.refine-factor", "8"); + + Map filtered = PaimonScanNode.filterVectorIndexOptions( + options, "embedding", "ivf-flat"); + + Assert.assertEquals(Collections.singletonMap("fields.embedding.refine-factor", "2"), filtered); + } @Test public void testRustReaderRejectsConflictingStaticCredentials() throws Exception { @@ -3540,6 +3769,66 @@ public void testRustMultiFilePrimaryKeyMergeFallback() throws Exception { } } + @Test + public void testRustMultiFileRawConvertibleSplitStaysOnRust() throws Exception { + // A rawConvertible split whose files are all compacted (level != 0) never reaches + // the pinned merge reader on BE: paimon-rust dispatches it to the raw path, which + // streams one file at a time bounded by read.batch-size (crate pk_split_needs_merge + // / read_pk). The multi-file merge gate must not judge it, or compacted PK buckets + // (the normal PK-vector table layout) could never use the rust reader at all. + for (String engine : Arrays.asList("deduplicate", "aggregation", "partial-update")) { + for (int fileCount : new int[] {2, 8}) { + for (boolean rawConvertible : Arrays.asList(true, false)) { + RustRoutingFixture f = new RustRoutingFixture(); + f.schema(1, new IntType(), ImmutableMap.of("merge-engine", engine, + "read.batch-size", "1"), true); + List files = new ArrayList<>(); + for (int i = 0; i < fileCount; i++) { + DataFileMeta file = Mockito.spy(createDataSplit("run-" + i + ".parquet") + .dataFiles().get(0)); + Mockito.doReturn(Optional.of(0L)).when(file).deleteRowCount(); + Mockito.doReturn(1).when(file).level(); + files.add(file); + } + DataSplit.Builder builder = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW) + .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0") + .withDataFiles(files); + if (rawConvertible) { + builder.rawConvertible(true); + } + DataSplit split = builder.build(); + Assert.assertEquals(engine + " files=" + fileCount + " raw=" + rawConvertible, + rawConvertible ? TPaimonReaderType.PAIMON_RUST : TPaimonReaderType.PAIMON_JNI, + f.reader(split)); + } + } + } + + // A level-0 file forces the merge path even on a rawConvertible split (deletion + // vector tables merge any uncompacted level), so the multi-file gate must keep + // routing it to JNI. + for (boolean rawConvertible : Arrays.asList(true, false)) { + RustRoutingFixture f = new RustRoutingFixture(); + f.schema(1, new IntType(), ImmutableMap.of("merge-engine", "deduplicate", + "deletion-vectors.enabled", "true", "read.batch-size", "1"), true); + DataFileMeta compacted = Mockito.spy(createDataSplit("compacted.parquet") + .dataFiles().get(0)); + Mockito.doReturn(Optional.of(0L)).when(compacted).deleteRowCount(); + Mockito.doReturn(1).when(compacted).level(); + DataFileMeta levelZero = Mockito.spy(createDataSplit("level0.parquet").dataFiles().get(0)); + Mockito.doReturn(Optional.of(0L)).when(levelZero).deleteRowCount(); + Mockito.doReturn(0).when(levelZero).level(); + DataSplit.Builder builder = DataSplit.builder().withPartition(BinaryRow.EMPTY_ROW) + .withBucket(0).withBucketPath("file:///warehouse/db/t/bucket-0") + .withDataFiles(Arrays.asList(compacted, levelZero)); + if (rawConvertible) { + builder.rawConvertible(true); + } + Assert.assertEquals("level0 raw=" + rawConvertible, + TPaimonReaderType.PAIMON_JNI, f.reader(builder.build())); + } + } + @Test public void testRustNestedEvolutionAndCompatibleHistory() throws Exception { org.apache.paimon.types.DataType small = new org.apache.paimon.types.TinyIntType(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java new file mode 100644 index 00000000000000..b01b2918ee69c1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoPaimonScanTest.java @@ -0,0 +1,1235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.Config; +import org.apache.doris.datasource.CatalogProperty; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.source.PaimonVectorSearch; +import org.apache.doris.datasource.property.metastore.MetastoreProperties; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProductApproximate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.L2DistanceApproximate; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.plans.AnnTopNInfo; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TVectorMetric; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import mockit.MockUp; +import org.apache.paimon.fs.Path; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.TimestampType; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Unit tests for {@link PushDownVectorTopNIntoPaimonScan}. + * + *

The rule turns {@code TopN[dis] -> Project[dis := dist_fn(vec, q)] -> [Filter] -> PaimonScan} + * into an index-only ANN scan that produces {@code __paimon_search_score_to_dis}, and replaces every + * occurrence of {@code dist_fn(vec, q)} with that column's slot. No arithmetic is emitted: the BE + * reader converts the score into the distance the function is defined to return while filling the + * block. These tests drive the two rules directly instead of going through the planner, so no + * catalog/BE is needed. + */ +public class PushDownVectorTopNIntoPaimonScanTest { + + private static final List QUERY_VECTOR = + ImmutableList.of(new FloatLiteral(1.0f), new FloatLiteral(2.0f)); + + private static SystemInfoService systemInfo; + private ConnectContext ctx; + private CascadesContext cascadesContext; + + @BeforeEach + public void setUp() { + // The rule reads enable_paimon_rust_reader / enable_file_scanner_v2 off the + // thread-local ConnectContext. Deliberately no StatementContext: + // MvccUtil.getSnapshotFromContext then short-circuits to Optional.empty(), + // so the mocked table never needs a database/catalog. + ctx = new ConnectContext(); + ctx.setSessionVariable(new SessionVariable()); + ctx.getSessionVariable().setEnablePaimonRustReader(true); + ctx.getSessionVariable().enableFileScannerV2 = true; + ctx.setThreadLocalInfo(); + // Rule.transform only stuffs the context into a MatchingContext; the rule body ignores it. + cascadesContext = Mockito.mock(CascadesContext.class); + // The rewrite's pre-flight (PaimonVectorSearch.vectorScanEligible) requires a + // rust-capable BE via the all-backend capability negotiation, mirroring + // PaimonScanNodeTest's routing fixture — and the VECTOR capability bit on + // top of the batch one, since an older rust-capable BE would reject the + // vector payload ("missing paimon_split"). + Backend rustBackend = GsonUtils.GSON.fromJson( + "{\"supportsPaimonRustReader\":true,\"supportsPaimonRustVectorReader\":true}", + Backend.class); + systemInfo = Mockito.mock(SystemInfoService.class); + Mockito.when(systemInfo.getAllClusterBackends(true)) + .thenReturn(ImmutableList.of(rustBackend)); + new MockUp() { + @mockit.Mock + public SystemInfoService getCurrentSystemInfo() { + return systemInfo; + } + }; + } + + @AfterEach + public void tearDown() { + ConnectContext.remove(); + } + + // ------------------------------------------------------------------ positive cases + + @Test + public void testL2AscPushedDown() { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), + dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + + // ANN info reached the scan, with the retrieval limit widened by the offset. + Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + AnnTopNInfo annTopN = newScan.getAnnTopN().get(); + Assertions.assertEquals("vec", annTopN.getColumnName()); + Assertions.assertEquals(15L, annTopN.getLimit()); + Assertions.assertArrayEquals(new float[] {1.0f, 2.0f}, annTopN.getQueryVector()); + // BE needs the metric to undo paimon-rust's score transform. + Assertions.assertEquals(TVectorMetric.L2, annTopN.getMetric()); + + // The distance column is appended as a virtual output column. + Assertions.assertEquals(1, newScan.getVirtualColumns().size()); + Assertions.assertEquals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN, + newScan.getVirtualColumns().get(0).getName()); + + // ... and it must actually reach the scan's OUTPUT, not just its virtualColumns + // list. These are two different things: output is computed once and can be + // served from a cache (LogicalProperties, or LogicalFileScan.cachedOutputs, + // which short-circuits computeOutput() entirely). withVectorTopN therefore has + // to invalidate both, and asserting only on getVirtualColumns() would pass even + // if the new column never became visible to the parent project -- which is what + // the plan actually consumes. + Assertions.assertTrue(newScan.getOutput().contains(distanceSlotOf(newScan).toSlot()), + () -> "distance slot missing from scan output: " + newScan.getOutput()); + Assertions.assertEquals(scan.getOutput().size() + 1, newScan.getOutput().size()); + + // The pre-existing base columns must keep their ExprIds. The rule reparents the + // EXISTING projections onto the new scan, so a regenerated ExprId is not a + // cosmetic renumbering -- it substitutes a different slot identity and leaves the + // project referencing something the scan no longer produces. This fails if + // withVectorTopN lets computeOutput() rebuild the output: computeOutput calls + // exprIdGenerator.getNextId() per column with no reuse cache. + Assertions.assertEquals( + scan.getOutput().stream().map(Slot::getExprId).collect(Collectors.toList()), + newScan.getOutput().subList(0, scan.getOutput().size()).stream() + .map(Slot::getExprId).collect(Collectors.toList()), + () -> "base column exprIds changed: " + scan.getOutput() + + " -> " + newScan.getOutput()); + + // The consequence the above protects, asserted directly: every slot the parent + // project reads has to be produced by the scan underneath it. + Set scanOutputIds = newScan.getOutput().stream() + .map(Slot::getExprId).collect(Collectors.toSet()); + for (NamedExpression projection : project.getProjects()) { + for (Slot input : projection.getInputSlots()) { + Assertions.assertTrue(scanOutputIds.contains(input.getExprId()), + () -> "project reads " + input + " which the scan does not output: " + + newScan.getOutput()); + } + } + + // `dis` keeps its alias identity (the TopN order key still resolves) ... + Alias newDis = (Alias) project.getProjects().get(1); + Assertions.assertEquals(dis.getExprId(), newDis.getExprId()); + Assertions.assertEquals("dis", newDis.getName()); + // ... and its value is the bare distance slot: the BE reader already converted it + // into the Euclidean distance, so no arithmetic is planned here. + Assertions.assertEquals(distanceSlotOf(newScan), newDis.child()); + Assertions.assertFalse(containsDistanceFunction(newDis)); + } + + @Test + public void testInnerProductDescPushedDown() { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "inner_product")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new InnerProductApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), + dis, false); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + Assertions.assertEquals(TVectorMetric.DOT_PRODUCT, newScan.getAnnTopN().get().getMetric()); + + // inner_product: the score IS the distance, so the alias wraps the bare distance slot. + Alias newDis = (Alias) project.getProjects().get(1); + Assertions.assertEquals(dis.getExprId(), newDis.getExprId()); + Assertions.assertEquals(distanceSlotOf(newScan), newDis.child()); + } + + @Test + public void testOutputColumnNameKeepsTheFunctionCallWithoutAlias() { + // SQL: SELECT id, l2_distance_approximate(vec, q) FROM xx ORDER BY l2_distance_approximate(vec, q) + // No AS alias, so the binder builds an Alias whose name comes from the child's toSql + // (nameFromChild=true). The result column header must be "l2_distance_approximate(...)", + // NOT "__paimon_search_score_to_dis" -- the user asked for a distance function, not an + // internal virtual column. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector())); + String expectedName = dis.getName(); + Assertions.assertTrue(dis.isNameFromChild()); + Assertions.assertTrue(expectedName.startsWith("l2_distance_approximate("), expectedName); + + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + Alias newDis = (Alias) project.getProjects().get(1); + + // The header keeps the original function text, and the value is the bare + // distance slot underneath it. + Assertions.assertEquals(expectedName, newDis.getName()); + Assertions.assertNotEquals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN, newDis.getName()); + Assertions.assertEquals(dis.getExprId(), newDis.getExprId()); + Assertions.assertEquals(distanceSlotOf(newScan), newDis.child()); + // The name was captured from the pre-rewrite child, so it must not have followed + // the child down to the distance slot. + Assertions.assertFalse(newDis.isNameFromChild()); + Assertions.assertEquals(expectedName, newDis.toSlot().getName()); + } + + @Test + public void testDistancePredicateFilterNotPushedDown() { + // WHERE l2_distance_approximate(vec, q) < 10 ORDER BY the same expression. + // A predicate on the distance cannot ride into the BE vector search (the + // synthetic distance column is not in the paimon-rust predicate converter's + // field registry), so it would run as a residual AFTER the bounded top-N + // fetch -- with candidate distances 0, 2 and 3, a `dis > 1 LIMIT 1` would + // fetch only the distance-0 row, filter it out and return nothing instead + // of the distance-2 row. An upper bound happens to be monotone in the + // ASC order, but the rewrite cannot rely on residual monotonicity, so the + // rewrite declines and the query computes the distance in Doris. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Expression distance = new L2DistanceApproximate(vec, queryVector()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(distance, new IntegerLiteral(10))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + assertNotPushedDown(topN); + } + + @Test + public void testSimpleColumnFilterStillPushedDown() { + // WHERE tag < 3 ORDER BY dis: a comparison on a plain supported-type column + // provably rides into the BE vector search (PaimonRustPredicateConverter + // pushes comparisons on BOOLEAN..BIGINT/STRING/DATE/DATEV2 columns), so the + // bounded top-N sees the predicate and the rewrite stays sound. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); + SlotReference vec = vecSlot(scan); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + Assertions.assertEquals("tag", tag.getName()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(tag, new IntegerLiteral(3))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + Plan rewritten = applyRule(1, topN); + Assertions.assertNotSame(topN, rewritten); + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) ((LogicalFilter) project.child(0)).child(0); + Assertions.assertTrue(newScan.getAnnTopN().isPresent(), + "the simple column filter must not block the bounded ANN rewrite"); + } + + @Test + public void testUnsupportedColumnTypeFilterNotPushedDown() { + // WHERE score < 1.5 with score being a DOUBLE column: the paimon-rust + // predicate converter keeps IEEE-ordering types (DOUBLE/FLOAT/DECIMAL/ + // VARCHAR/DATETIME) residual, and a residual after the bounded top-N is + // unsound, so the rewrite declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference score = (SlotReference) scan.getOutput().get(4); + Assertions.assertEquals("score", score.getName()); + Assertions.assertEquals(Type.DOUBLE, score.getOriginalColumn().get().getType()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(score, new FloatLiteral(1.5f))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + assertNotPushedDown(topN); + } + + @Test + public void testExpressionFilterNotPushedDown() { + // WHERE id + 1 < 10: an arithmetic expression cannot ride into the vector + // search, so the rewrite declines instead of filtering after the limit. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference id = idSlot(scan); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan( + new Add(id, new IntegerLiteral(1)), new IntegerLiteral(10))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + assertNotPushedDown(topN); + } + + @Test + public void testNestedProjectionOccurrenceReusesScore() { + // A second projection embeds the same distance call: dis2 := dist_fn(vec, q) * 2. + // The nested occurrence is replaced while the surrounding alias keeps its identity. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + Alias doubled = new Alias( + new Multiply(new L2DistanceApproximate(vec, queryVector()), new IntegerLiteral(2)), + "dis2"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis, doubled), scan), dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + Alias newDoubled = (Alias) project.getProjects().get(2); + Assertions.assertEquals(doubled.getExprId(), newDoubled.getExprId()); + Assertions.assertEquals("dis2", newDoubled.getName()); + Assertions.assertEquals( + new Multiply(distanceSlotOf(newScan), new IntegerLiteral(2)), + newDoubled.child()); + Assertions.assertFalse(containsDistanceFunction(newDoubled)); + } + + @Test + public void testSelectStarDoesNotSurfaceDistanceColumn() { + // SELECT * FROM xx ORDER BY l2_distance_approximate(vec, q) + // The distance column must not appear in the result set. The rule appends it + // to the scan output but deliberately does NOT extend the asterisk output, + // so a downstream `SELECT *` expansion never picks it up. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), vecSlot(scan), dis), scan), + dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + + // The scan output has the distance column appended ... + Assertions.assertTrue(newScan.getOutput().stream() + .anyMatch(s -> s.getName().equals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)), + "distance slot must be in scan output"); + + // ... but the asterisk output must NOT contain it. + List asterisk = newScan.getLogicalProperties().getAsteriskOutput(); + boolean distanceInAsterisk = asterisk.stream() + .anyMatch(s -> s.getName().equals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)); + Assertions.assertFalse(distanceInAsterisk, + "__paimon_search_score_to_dis must not appear in SELECT * output: " + asterisk); + + // The project also must not leak the internal column name as a visible output. + for (NamedExpression proj : project.getProjects()) { + Assertions.assertNotEquals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN, proj.getName(), + "project output must not expose internal distance column name: " + proj); + } + } + + @Test + public void testAliasedOutputColumnNameKeepsUserAlias() { + // SQL: SELECT id, l2_distance_approximate(vec, q) AS dis FROM xx ORDER BY dis + // The result column header must be "dis", NOT "__paimon_search_score_to_dis". + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true); + + Plan rewritten = applyRule(0, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFileScan newScan = (LogicalFileScan) project.child(0); + + Alias rewrittenDis = (Alias) project.getProjects().get(1); + // The alias name stays "dis", not the internal column name. + Assertions.assertEquals("dis", rewrittenDis.getName()); + Assertions.assertNotEquals(PaimonVectorSearch.SEARCH_DISTANCE_COLUMN, rewrittenDis.getName()); + // The alias still wraps the bare distance slot (no arithmetic). + Assertions.assertEquals(distanceSlotOf(newScan), rewrittenDis.child()); + Assertions.assertFalse(containsDistanceFunction(rewrittenDis)); + // The slot produced by the alias also carries the user-visible name. + Assertions.assertEquals("dis", rewrittenDis.toSlot().getName()); + } + + @Test + public void testPredicateOnAliasedDistanceNotPushedDown() { + // SELECT id, l2_distance_approximate(vec, q) AS dis + // FROM xx WHERE l2_distance_approximate(vec, q) < 1 ORDER BY dis + // The predicate is on the distance: it cannot ride into the BE vector + // search (the synthetic distance column is outside the paimon-rust + // predicate converter's field registry), and applying it as a residual + // after the bounded top-N fetch is unsound, so the rewrite declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Expression distanceExpr = new L2DistanceApproximate(vec, queryVector()); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(distanceExpr, new FloatLiteral(1.0f))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + assertNotPushedDown(topN); + } + + @Test + public void testMergedSubqueryRewriteStillTriggers() { + // SELECT id, dis FROM ( + // SELECT id, l2_distance_approximate(vec, q) AS dis FROM xx + // ) WHERE tag < 3 ORDER BY dis + // + // After MergeProjects / MergeFilters runs in the planner, the inner and outer + // projects collapse. The rule then sees: + // TopN -> Project[id, dis := l2_distance_approximate(vec,q)] + // -> Filter[tag < 3] + // -> PaimonScan + // which matches the second rule variant (with filter). The simple column + // comparison rides into the vector search, so the rewrite stays sound. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); + SlotReference vec = vecSlot(scan); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(tag, new IntegerLiteral(3))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + LogicalTopN topN = topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true); + + Plan rewritten = applyRule(1, topN); + Assertions.assertNotSame(topN, rewritten); + + LogicalProject project = (LogicalProject) rewritten.child(0); + LogicalFilter newFilter = (LogicalFilter) project.child(0); + LogicalFileScan newScan = (LogicalFileScan) newFilter.child(0); + + // ANN rewrite fired. + Assertions.assertTrue(newScan.getAnnTopN().isPresent()); + Assertions.assertEquals(TVectorMetric.L2, newScan.getAnnTopN().get().getMetric()); + + // The filter keeps its column predicate (nothing to replace). + Expression conjunct = newFilter.getConjuncts().iterator().next(); + Assertions.assertFalse(containsDistanceFunction(conjunct)); + Assertions.assertEquals(new LessThan(tag, new IntegerLiteral(3)), conjunct); + + // The alias keeps its user name "dis" and wraps the distance slot. + Alias rewrittenDis = (Alias) project.getProjects().get(1); + Assertions.assertEquals("dis", rewrittenDis.getName()); + Assertions.assertEquals(distanceSlotOf(newScan), rewrittenDis.child()); + } + + // ------------------------------------------------------------------ negative cases + + @Test + public void testRustReaderDisabledNotPushedDown() { + // Only the BE paimon-rust reader can execute the index-only search. + ctx.getSessionVariable().setEnablePaimonRustReader(false); + assertNotPushedDown(l2Plan("l2", true)); + } + + @Test + public void testFileScannerV2DisabledNotPushedDown() { + // FileScannerV1 cannot select the paimon-rust reader, so an ANN request would be + // one the selected scanner cannot consume. + ctx.getSessionVariable().enableFileScannerV2 = false; + assertNotPushedDown(l2Plan("l2", true)); + } + + @Test + public void testMetricMismatchNotPushedDown() { + // inner_product_approximate over an l2 index would return a different ranking. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + Alias dis = new Alias(new InnerProductApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, false)); + } + + @Test + public void testL2DescNotPushedDown() { + // l2 top-N is only an ANN search when ordered ascending (nearest first). + assertNotPushedDown(l2Plan("l2", false)); + } + + @Test + public void testInnerProductAscNotPushedDown() { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "inner_product")); + Alias dis = new Alias(new InnerProductApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testCosineMetricNotPushedDown() { + // v1 supports l2 / inner_product only. + assertNotPushedDown(l2Plan("cosine", true)); + } + + @Test + public void testNoVectorIndexNotPushedDown() { + LogicalFileScan scan = paimonScan(ImmutableMap.of()); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testUnindexedColumnNotPushedDown() { + // The index covers `vec`, but the query orders by a distance over `vec2`. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec2 = (SlotReference) scan.getOutput().get(2); + Assertions.assertEquals("vec2", vec2.getName()); + Alias dis = new Alias(new L2DistanceApproximate(vec2, queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testNonConstantQueryVectorNotPushedDown() { + // The query vector must be a literal to be handed to the index. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference vec2 = (SlotReference) scan.getOutput().get(2); + Alias dis = new Alias(new L2DistanceApproximate(vec, vec2), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testOrderKeyNotFromProjectionNotPushedDown() { + // Ordering by a plain column is not a vector top-N. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference id = idSlot(scan); + LogicalProject project = new LogicalProject<>(ImmutableList.of(id), scan); + assertNotPushedDown(new LogicalTopN<>( + ImmutableList.of(new OrderKey(id, true, true)), 10, 5, project)); + } + + @Test + public void testIncrementalScanNotPushedDown() { + // t@incr(...): the PK-vector search path has no incremental semantics (the + // VectorScan builder has no incremental mode), so an incremental query must + // keep its ordinary scan and compute the distance in Doris — a rewritten + // vector split would silently return snapshot rows for a query that asked + // for changes. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.of(new TableScanParams("incr", + ImmutableMap.of("start", "1"), ImmutableList.of()))); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testFileCreationTimeScanNotPushedDown() { + // t@options('scan.file-creation-time-millis'='...'): the ordinary path drops + // manifest entries older than the bound, but the PK-vector search plans its + // own splits and never consults that bound -- old files' nearest neighbors + // would leak into the bounded result. Snapshot/tag selectors are resolved + // into the table before the vector search and stay eligible; only the + // file-level bound declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.of(new TableScanParams("options", + ImmutableMap.of("scan.file-creation-time-millis", "1791793167062"), + ImmutableList.of()))); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testCreationTimeScanNotPushedDown() { + // t@options('scan.creation-time-millis'='...'): when a snapshot exists past + // the bound this resolves into an ordinary snapshot pin (safe), but when none + // does it resolves into the SAME internal file-creation bound the vector + // search never consults — which case applies depends on the snapshot + // manager's state, so the raw key declines unconditionally (contrast with + // scan.snapshot-id, which stays eligible). + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.of(new TableScanParams("options", + ImmutableMap.of("scan.creation-time-millis", "1791793167062"), + ImmutableList.of()))); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testNullQueryVectorElementNotPushedDown() { + // [CAST(NULL AS FLOAT), 0.0]: a NullLiteral's getDouble() is 0, so the + // rewrite would silently search a zero vector and return rows for a + // different input, while the ordinary path raises (the distance functions + // reject NULL elements). Any null element declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + ArrayLiteral nullVector = new ArrayLiteral(ImmutableList.of( + new NullLiteral(), new FloatLiteral(2.0f))); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), nullVector), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testInPredicateWithNullOptionNotPushedDown() { + // tag IN (1, 2, NULL): every option is a Literal so the shape whitelist + // passes, but the BE converter rejects the null literal and with it the + // WHOLE predicate, leaving the IN in the post-fetch residual — the bounded + // fetch then returns the wrong rows. A null option never rides. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + LogicalFilter filter = new LogicalFilter<>(ImmutableSet.of( + new InPredicate(tag, ImmutableList.of(new IntegerLiteral(1), new IntegerLiteral(2), + new NullLiteral()))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true)); + } + + @Test + public void testInPredicateStillPushedDown() { + // Control for the NULL-option case: tag IN (1, 2) with every option a + // non-null literal on a supported column rides into the vector search. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + LogicalFilter filter = new LogicalFilter<>(ImmutableSet.of( + new InPredicate(tag, ImmutableList.of(new IntegerLiteral(1), new IntegerLiteral(2)))), + scan); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + Plan rewritten = applyRule(1, topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true)); + Assertions.assertNotSame(scan, ((LogicalFileScan) ((LogicalFilter) + ((LogicalProject) rewritten.child(0)).child(0)).child(0))); + Assertions.assertTrue(((LogicalFileScan) ((LogicalFilter) + ((LogicalProject) rewritten.child(0)).child(0)).child(0)).getAnnTopN().isPresent()); + } + + @Test + public void testFilterWithoutDeletionVectorsNotPushedDown() { + // WHERE tag < 3 on a table without deletion vectors: the predicate + // converts fine, but paimon-rust's PkVectorSearchParams::resolve rejects + // every data predicate unless deletion vectors are enabled and + // merge-on-read is disabled — so the failure would land AFTER the + // one-way rewrite, as a user-facing error on a query the ordinary path + // answers. Decline instead (first-row tables allow the index without DV). + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference tag = (SlotReference) scan.getOutput().get(3); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(new LessThan(tag, new IntegerLiteral(3))), scan); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true)); + } + + @Test + public void testSyntheticDistanceColumnCollisionNotPushedDown() { + // __paimon_search_score_to_dis is also a legal paimon user column name; a + // table that HAS such a column collides with the synthetic slot's + // block-column matching, so the rewrite declines. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.empty(), + ImmutableList.of("id", "vec", PaimonVectorSearch.SEARCH_DISTANCE_COLUMN)); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testPrunedComplexProjectionNotPushedDown() { + // Project(pmap['k'], dis): nested pruning materializes the scan tuple with + // a REDUCED type for pmap, but the vector reader projects only root names + // and returns the full Arrow value, whose SerDe binds children by ordinal — + // the wrong child comes back. Only a whole-column (bare slot) projection + // keeps the rewrite sound. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference pmap = (SlotReference) scan.getOutput().get(6); + Assertions.assertEquals("pmap", pmap.getName()); + Assertions.assertTrue(pmap.getDataType().isComplexType()); + Alias field = new Alias(new ElementAt(pmap, new StringLiteral("k")), "pk"); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), field, dis), scan), dis, true)); + } + + @Test + public void testWholeComplexProjectionStillPushedDown() { + // Control for the pruned case: the complex column rides as a whole column + // (bare slot output), which nested pruning leaves untouched. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference pmap = (SlotReference) scan.getOutput().get(6); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + Plan rewritten = applyRule(0, topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), pmap, dis), scan), dis, true)); + Assertions.assertNotSame(scan, ((LogicalFileScan) ((LogicalProject) rewritten.child(0)).child(0))); + Assertions.assertTrue(((LogicalFileScan) ((LogicalProject) rewritten.child(0)).child(0)) + .getAnnTopN().isPresent()); + } + + @Test + public void testBackendWithoutVectorCapabilityNotPushedDown() { + // Rolling upgrade: a BE that reports the batch rust bit but not the vector + // bit would reject the rewritten payload ("missing paimon_split"), so the + // rewrite must decline and leave the query on the ordinary path it can + // still run. Re-stub the shared SystemInfoService with such a BE. + Backend legacyBackend = GsonUtils.GSON.fromJson( + "{\"supportsPaimonRustReader\":true}", Backend.class); + Mockito.when(systemInfo.getAllClusterBackends(true)) + .thenReturn(ImmutableList.of(legacyBackend)); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testSnapshotScanStillPushedDown() { + // t@options('scan.snapshot-id'='1'): snapshot selectors are resolved into + // the table handle before the vector search plans its splits, so they do + // not block the rewrite (contrast with the file-creation-time bound). + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), + Optional.of(new TableScanParams("options", + ImmutableMap.of("scan.snapshot-id", "1"), ImmutableList.of()))); + SlotReference vec = vecSlot(scan); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + Plan rewritten = applyRule(0, topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + Assertions.assertNotSame(scan, ((LogicalProject) rewritten.child(0)).child(0)); + Assertions.assertTrue(((LogicalFileScan) ((LogicalProject) rewritten.child(0)) + .child(0)).getAnnTopN().isPresent()); + } + + @Test + public void testCastOnIndexedVectorNotPushedDown() { + // l2_distance_approximate(CAST(vec AS ARRAY), q): a legal cast chain + // can change vector values (an ARRAY -> ARRAY -> ARRAY + // round trip loses precision), so searching the original vectors while the + // query means the casted ones would change the ranking and the distance. + // The rewrite only accepts a direct indexed column slot. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + Cast castVec = new Cast(vec, org.apache.doris.nereids.types.ArrayType.of( + org.apache.doris.nereids.types.DoubleType.INSTANCE)); + Alias dis = new Alias(new L2DistanceApproximate(castVec, queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testVariantProjectionNotPushedDown() { + // SELECT payload, dis ... ORDER BY dis with a VARIANT payload column: the + // rust reader has no Variant decoder, so the scan node's projectedVariant + // bound would reject the vector split -- and a vector split has no JNI + // fallback, so the rewrite would turn a supported query into an error. + // Pre-flight the projection and decline instead; the query then scans + // through the ordinary path. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference vec = vecSlot(scan); + SlotReference payload = (SlotReference) scan.getOutput().get(5); + Assertions.assertEquals("payload", payload.getName()); + Assertions.assertTrue(payload.getOriginalColumn().get().getType().isVariantType()); + Alias dis = new Alias(new L2DistanceApproximate(vec, queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), payload, dis), scan), dis, true)); + } + + @Test + public void testBranchTableWithDifferentMetricNotPushedDown() { + // t@branch(b): the relation's bound table is the branch, which can carry an + // independently configured index metric. The pre-flight must validate the + // BOUND table: an l2 query against an inner-product branch would score with + // the branch's metric and reorder with l2's inverse -- wrong results even + // at full recall. + // field names of the fixture schema (see below): the branch table's rowType + // drives the same synthetic-distance-collision pre-flight as the main table. + List fieldNames = ImmutableList.of("id", "vec", "vec2", "tag", "score", + "payload", "pmap"); + FileStoreTable branch = mockFileStoreTable(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "inner_product"), fieldNames, null); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), Optional.empty(), null, false, + null, branch); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testBranchTableWithMatchingMetricStillPushedDown() { + // Control: the bound branch table carries the same l2 metric, so the + // rewrite must fire exactly as it does against the main table. + List fieldNames = ImmutableList.of("id", "vec", "vec2", "tag", "score", + "payload", "pmap"); + FileStoreTable branch = mockFileStoreTable(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), fieldNames, null); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), Optional.empty(), null, false, + null, branch); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + Plan rewritten = applyRule(0, topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + Assertions.assertTrue(((LogicalFileScan) ((LogicalProject) rewritten.child(0)) + .child(0)).getAnnTopN().isPresent()); + } + + @Test + public void testHistoricalSchemaEvolutionNotPushedDown() { + // A TIMESTAMP(3) -> TIMESTAMP(9) widening with old files retained: the + // vector reader's Arrow safe cast NULLs out-of-range values the ordinary + // path preserves, so the rewrite must decline and let the query run + // ordinary (see PaimonRustReaderCapabilities.hasRustIncompatibleSchemaHistory). + TableSchema widened = new TableSchema(4, + Collections.singletonList(new DataField(0, "ts", new TimestampType(9))), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null); + TableSchema millis = new TableSchema(5, + Collections.singletonList(new DataField(0, "ts", new TimestampType(3))), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null); + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2"), Optional.empty(), null, false, + ImmutableList.of(widened, millis), null); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, true)); + } + + @Test + public void testNonMovableProjectExpressionNotPushedDown() { + // c := assert_true(id > 0): on the ordinary path the Project evaluates before + // the TopN and raises for a far row the bounded fetch would never deliver; + // withVectorTopN truncates first, so the assertion never fires and the query + // silently succeeds. A NoneMovableFunction in the projection keeps the + // ordinary path. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + Alias guard = new Alias( + new AssertTrue(new GreaterThan(idSlot(scan), new IntegerLiteral(0)), + new StringLiteral("bad id")), + "c"); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN(new LogicalProject<>( + ImmutableList.of(idSlot(scan), guard, dis), scan), dis, true)); + } + + @Test + public void testPrunedBareComplexSlotNotPushedDown() { + // An outer field projection can leave an already-pruned bare slot inside the + // ANN Project (nested pruning materialized its type to the accessed subset). + // The desc-side backstop would fail loudly, so the pre-flight declines: a + // bare slot only rides when its type still equals the original column's. + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2")); + SlotReference pmap = (SlotReference) scan.getOutput().get(6); + Assertions.assertEquals("pmap", pmap.getName()); + SlotReference pruned = new SlotReference( + StatementScopeIdGenerator.getExprIdGenerator().getNextId(), "pmap", + org.apache.doris.nereids.types.ArrayType.of( + org.apache.doris.nereids.types.DoubleType.INSTANCE), + pmap.nullable(), ImmutableList.of(), scan.getTable(), + pmap.getOriginalColumn().get(), scan.getTable(), + pmap.getOriginalColumn().get(), ImmutableList.of()); + Alias prunedAlias = new Alias(pruned, "p"); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN(new LogicalProject<>( + ImmutableList.of(idSlot(scan), prunedAlias, dis), scan), dis, true)); + } + + @Test + public void testInPredicateBeyondConverterCapNotPushedDown() { + // With expr_children_limit raised, an IN can exceed the BE rust predicate + // converter's 16-bit child count: the converter drops the whole predicate, + // and the residual re-applies it AFTER the bounded Top-K — the truncation + // the whitelist exists to prevent. Options whose slot+options children + // exceed the cap keep the ordinary path. + int originalChildrenLimit = Config.expr_children_limit; + Config.expr_children_limit = 0xFFFF + 64; + LogicalFileScan scan; + SlotReference tag; + try { + scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", "l2", + "deletion-vectors.enabled", "true")); + tag = (SlotReference) scan.getOutput().get(3); + List options = new java.util.ArrayList<>(); + for (int i = 0; i < 0xFFFF; i++) { + options.add(new IntegerLiteral(i)); + } + LogicalFilter filter = + new LogicalFilter<>(ImmutableSet.of(new InPredicate(tag, options)), scan); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + assertNotPushedDown(topN( + new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), filter), dis, true)); + } finally { + Config.expr_children_limit = originalChildrenLimit; + } + } + + // ------------------------------------------------------------------ helpers + + private Plan applyRule(int ruleIndex, LogicalTopN topN) { + Rule rule = new PushDownVectorTopNIntoPaimonScan().buildRules().get(ruleIndex); + List result = rule.transform(topN, cascadesContext); + Assertions.assertEquals(1, result.size()); + return result.get(0); + } + + /** A no-op rule returns the origin plan unchanged (see PatternMatcher#toRule). */ + private void assertNotPushedDown(LogicalTopN topN) { + int ruleIndex = topN.child().child(0) instanceof LogicalFilter ? 1 : 0; + Assertions.assertSame(topN, applyRule(ruleIndex, topN)); + } + + private LogicalTopN l2Plan(String metric, boolean asc) { + LogicalFileScan scan = paimonScan(ImmutableMap.of( + "pk-vector.index.columns", "vec", + "fields.vec.pk-vector.distance.metric", metric)); + Alias dis = new Alias(new L2DistanceApproximate(vecSlot(scan), queryVector()), "dis"); + return topN(new LogicalProject<>(ImmutableList.of(idSlot(scan), dis), scan), dis, asc); + } + + private LogicalTopN topN(LogicalProject project, Alias orderBy, boolean asc) { + // limit 10 offset 5 -> the scan must retrieve 15 candidates. + return new LogicalTopN<>(ImmutableList.of(new OrderKey(orderBy.toSlot(), asc, asc)), + 10, 5, project); + } + + /** + * A scan over a mocked Paimon table {@code (id INT, vec ARRAY, vec2 ARRAY)} + * whose Paimon options are exactly {@code options}. + */ + private LogicalFileScan paimonScan(Map options) { + return paimonScan(options, Optional.empty()); + } + + private LogicalFileScan paimonScan(Map options, + Optional scanParams) { + return paimonScan(options, scanParams, null); + } + + /** {@code paimonFieldNames} overrides the schema-derived names the collision + * pre-flight reads off the mocked paimon rowType. */ + private LogicalFileScan paimonScan(Map options, + Optional scanParams, List paimonFieldNames) { + return paimonScan(options, scanParams, paimonFieldNames, false); + } + + /** {@code nullableVector} makes the fixture's vec column nullable (the stored-NULL + * contract tests). */ + private LogicalFileScan paimonScan(Map options, + Optional scanParams, List paimonFieldNames, + boolean nullableVector) { + return paimonScan(options, scanParams, paimonFieldNames, nullableVector, null, null); + } + + /** + * Full fixture. {@code extraHistory} appends rust-incompatible historical schemas + * to the bound table's schema history (evolution tests); {@code boundTable} binds + * a branch-style relation snapshot whose getPaimonTable resolves to that table — + * the main-table stub stays for the anything-else lookups. + */ + private LogicalFileScan paimonScan(Map options, + Optional scanParams, List paimonFieldNames, + boolean nullableVector, List extraHistory, + org.apache.paimon.table.Table boundTable) { + List schema = ImmutableList.of( + new Column("id", Type.INT, true), + new Column("vec", new ArrayType(Type.FLOAT), nullableVector), + new Column("vec2", new ArrayType(Type.FLOAT), nullableVector), + new Column("tag", Type.INT, true), + new Column("score", Type.DOUBLE, true), + new Column("payload", Type.VARIANT, true), + // A complex column the pruned-projection tests access with + // ElementAt (the former struct_element): only the isComplexType() + // verdict matters to the pre-flight, so a Doris MAP stands in for + // a STRUCT this fixture cannot express through catalog types. + new Column("pmap", new org.apache.doris.catalog.MapType(), true)); + if (paimonFieldNames == null) { + paimonFieldNames = schema.stream().map(Column::getName).collect(Collectors.toList()); + } + + PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); + Mockito.when(table.getName()).thenReturn("paimon_tbl"); + Mockito.when(table.getBaseSchema()).thenReturn(schema); + Mockito.when(table.getFullSchema()).thenReturn(schema); + Mockito.when(table.getFullSchema(Mockito.>any())) + .thenReturn(schema); + Mockito.when(table.initSelectedPartitions(Mockito.any())) + .thenReturn(SelectedPartitions.NOT_PRUNED); + + FileStoreTable paimonTable = mockFileStoreTable(options, paimonFieldNames, extraHistory); + Mockito.when(table.getPaimonTable(Mockito.>any())) + .thenReturn(paimonTable); + Mockito.when(table.getPaimonTable(Mockito.any())).thenReturn(paimonTable); + // An options-type TableScanParams binds its schema through + // getFullSchema(TableScanParams) (the relation-option path inside + // PaimonExternalTable, which would otherwise hit the unmocked package + // private helpers). Stub it directly so the options tests get real + // output slots; the rule itself only reads the params' raw map. + Mockito.when(table.getFullSchema(Mockito.any())).thenReturn(schema); + + Optional relationSnapshot = Optional.empty(); + if (boundTable != null) { + // Stub AFTER the any() stub: Mockito resolves the last matching stub, + // so this relation's snapshot resolves to the branch/bound table while + // everything else keeps the main table. + org.apache.doris.datasource.mvcc.MvccSnapshot snapshot = + Mockito.mock(org.apache.doris.datasource.mvcc.MvccSnapshot.class); + Mockito.when(table.getPaimonTable(Mockito.eq(Optional.of(snapshot)))) + .thenReturn(boundTable); + relationSnapshot = Optional.of(snapshot); + } + + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty); + // A filesystem metastore (empty properties, benign type): no vended-credential + // provider, so the factory returns the base (empty) storage map. + MetastoreProperties metastoreProperties = Mockito.mock(MetastoreProperties.class); + Mockito.when(metastoreProperties.getType()).thenReturn(MetastoreProperties.Type.FILE_SYSTEM); + Mockito.when(catalogProperty.getMetastoreProperties()).thenReturn(metastoreProperties); + Mockito.when(catalogProperty.getStoragePropertiesMap()) + .thenReturn(java.util.Collections.emptyMap()); + + return new LogicalFileScan(StatementScopeIdGenerator.newRelationId(), table, + ImmutableList.of("paimon_ctl", "paimon_db"), ImmutableList.of(), + Optional.empty(), Optional.empty(), scanParams, Optional.empty(), + relationSnapshot); + } + + /** A FileStoreTable mock: local file:// location, no REST FileIO, and the + * per-test options map as the only input the pre-flight gates see. + * {@code extraHistory} appends historical schemas to the schema-manager's + * listAll, so evolution tests can supply rust-incompatible versions; + * null means an empty history (only the current schema). */ + private FileStoreTable mockFileStoreTable(Map options, + List paimonFieldNames, List extraHistory) { + FileStoreTable paimonTable = Mockito.mock(FileStoreTable.class); + Mockito.when(paimonTable.options()).thenReturn(options); + Mockito.when(paimonTable.location()) + .thenReturn(new Path("file:///warehouse/wh/paimon_db.db/paimon_tbl")); + // The schema-history pre-flight: a current schema whose id is alone in the + // history means no evolution, so the positive cases pass it. + TableSchema currentSchema = new TableSchema( + 5, Collections.singletonList(new DataField(0, "id", new IntType())), + 0, Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap(), null); + Mockito.when(paimonTable.schema()).thenReturn(currentSchema); + List history = new java.util.ArrayList<>(); + history.add(currentSchema); + if (extraHistory != null) { + history.addAll(extraHistory); + } + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + Mockito.when(schemaManager.listAll()).thenReturn(history); + Mockito.when(paimonTable.schemaManager()).thenReturn(schemaManager); + // rowType drives the synthetic-distance-column collision pre-flight; its + // field names track the fixture schema (or the test's explicit names). + org.apache.paimon.types.RowType rowType = Mockito.mock( + org.apache.paimon.types.RowType.class); + Mockito.when(rowType.getFieldNames()).thenReturn(paimonFieldNames); + Mockito.when(paimonTable.rowType()).thenReturn(rowType); + Mockito.when(paimonTable.fileIO()).thenReturn(null); + Mockito.when(paimonTable.coreOptions()).thenReturn(null); + return paimonTable; + } + + private SlotReference idSlot(LogicalFileScan scan) { + return (SlotReference) scan.getOutput().get(0); + } + + /** Must come from the scan output: the rule requires the slot's original column/table. */ + private SlotReference vecSlot(LogicalFileScan scan) { + return (SlotReference) scan.getOutput().get(1); + } + + private ArrayLiteral queryVector() { + return new ArrayLiteral(QUERY_VECTOR); + } + + private NamedExpression distanceSlotOf(LogicalFileScan scan) { + return scan.getVirtualColumns().get(0); + } + + private boolean containsDistanceFunction(Expression expression) { + return expression.anyMatch(e -> + e instanceof L2DistanceApproximate || e instanceof InnerProductApproximate); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java index 85ff507ddb60c0..1ec6586bfbe452 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PaimonPredicateConverterTest.java @@ -17,8 +17,12 @@ package org.apache.doris.planner; +import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.InPredicate; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.paimon.source.PaimonPredicateConverter; import org.apache.doris.qe.StmtExecutor; @@ -152,4 +156,40 @@ public void equal() throws Exception { Assertions.assertTrue(predicate.children().get(0) instanceof LeafPredicate); Assertions.assertTrue(predicate.children().get(1) instanceof LeafPredicate); } + + /** + * A conjunct on a column the Paimon table does not have must be skipped, not fatal. + * + *

The PK-vector rewrite turns {@code dist_fn(vec, q) < x} into a comparison on the + * synthetic {@code __paimon_search_score_to_dis} slot, which exists only in the scan's + * output. Before the guard, the field lookup returned -1 and the following + * {@code paimonFieldTypes.get(-1)} threw IndexOutOfBounds, killing the query during + * planning. Both entry points are covered: binary comparison and IN. + */ + @Test + public void unknownColumnIsSkippedInsteadOfThrowing() { + RowType rowType = new RowType(Lists.newArrayList( + new DataField(0, "k1", new IntType()), + new DataField(1, "k2", new IntType()))); + PaimonPredicateConverter converter = new PaimonPredicateConverter(rowType); + + SlotRef distanceSlot = new SlotRef(null, "__paimon_search_score_to_dis"); + Expr lessThan = new BinaryPredicate(BinaryPredicate.Operator.LT, distanceSlot, + new IntLiteral(1)); + Assertions.assertTrue( + converter.convertToPaimonExpr(Lists.newArrayList(lessThan)).isEmpty()); + + Expr inPredicate = new InPredicate(distanceSlot, + Lists.newArrayList(new IntLiteral(1), new IntLiteral(2)), false); + Assertions.assertTrue( + converter.convertToPaimonExpr(Lists.newArrayList(inPredicate)).isEmpty()); + + // A known column still converts, so the guard did not disable push-down wholesale. + Expr knownColumn = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new SlotRef(null, "k1"), new IntLiteral(1)); + List converted = + converter.convertToPaimonExpr(Lists.newArrayList(knownColumn)); + Assertions.assertEquals(1, converted.size()); + Assertions.assertEquals("k1", ((LeafPredicate) converted.get(0)).fieldName()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java index 9b085b3c7e4a99..ae50e66f89b828 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/system/BackendPaimonRustCapabilityTest.java @@ -63,6 +63,43 @@ public void testLegacyHeartbeatClearsPreviouslyAdvertisedSupport() { Assert.assertFalse(GsonUtils.GSON.fromJson("{}", Backend.class).isPaimonRustReaderSupported()); } + @Test + public void testVectorCapabilityNegotiatedIndependentlyOfBatchSupport() { + // Rolling upgrade: a BE advertising only the batch rust bit keeps batch + // support and loses the vector protocol — the one-way vector rewrite must + // be gated on this bit specifically (an older rust-capable BE would reject + // the rewritten payload as "missing paimon_split"). + Backend backend = new Backend(1, "127.0.0.1", 9050); + BackendHbResponse batchOnly = heartbeat(); + batchOnly.setPaimonRustReaderSupported(true); + Assert.assertFalse(batchOnly.isPaimonRustVectorReaderSupported()); + backend.handleHbResponse(batchOnly, false); + Assert.assertTrue(backend.isPaimonRustReaderSupported()); + Assert.assertFalse(backend.isPaimonRustVectorReaderSupported()); + + BackendHbResponse bothBits = heartbeat(); + bothBits.setPaimonRustReaderSupported(true); + bothBits.setPaimonRustVectorReaderSupported(true); + BackendHbResponse replay = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(bothBits), BackendHbResponse.class); + Assert.assertTrue(replay.isPaimonRustVectorReaderSupported()); + backend.handleHbResponse(replay, false); + Assert.assertTrue(backend.isPaimonRustReaderSupported()); + Assert.assertTrue(backend.isPaimonRustVectorReaderSupported()); + Backend restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(backend), Backend.class); + Assert.assertTrue(restored.isPaimonRustVectorReaderSupported()); + + // A later heartbeat without the vector bit (downgrade) clears it again. + BackendHbResponse downgrade = heartbeat(); + downgrade.setPaimonRustReaderSupported(true); + backend.handleHbResponse(downgrade, false); + Assert.assertTrue(backend.isPaimonRustReaderSupported()); + Assert.assertFalse(backend.isPaimonRustVectorReaderSupported()); + + TBackendInfo thrift = new TBackendInfo(); + Assert.assertFalse(thrift.isSetSupportsPaimonRustVectorReader()); + Assert.assertFalse(GsonUtils.GSON.fromJson("{}", Backend.class).isPaimonRustVectorReaderSupported()); + } + private BackendHbResponse heartbeat() { return new BackendHbResponse(1, 9060, 8040, 8060, 1000, 1000, "test-version", Tag.VALUE_MIX, 0, 0, false, 8070); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java index f6752074f0a5f3..44469c9df69f4c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java @@ -20,9 +20,6 @@ import org.apache.doris.datasource.paimon.PaimonReaderOptions; import org.apache.paimon.CoreOptions; -import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.privilege.PrivilegeChecker; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; @@ -74,10 +71,8 @@ void testReadOptimizedTableKeepsFallbackPairAsDirectSource() throws Exception { Mockito.when(main.options()).thenReturn(Collections.emptyMap()); Mockito.when(fallback.options()).thenReturn(Collections.emptyMap()); FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main, fallback, true); - FileStoreTable privileged = PrivilegedFileStoreTable.wrap( - pair, Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "table")); - Table systemTable = PaimonTableValuedFunction.createRuntimeSafeSystemTable(privileged, "ro"); + Table systemTable = PaimonTableValuedFunction.createRuntimeSafeSystemTable(pair, "ro"); Assertions.assertInstanceOf(ReadOptimizedTable.class, systemTable); Field wrapped = ReadOptimizedTable.class.getDeclaredField("wrapped"); diff --git a/fe/pom.xml b/fe/pom.xml index 556c47639a6320..f1550b2ed2da3f 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -231,7 +231,7 @@ under the License. fe-authentication - 3.1.3 + 4.1.0 1.12.1 1.18.0 @@ -421,7 +421,7 @@ under the License. 2.3.2 2.0.3 - 1.4.2 + 2.1.0 3.4.4 shade-format-flatbuffers diff --git a/gensrc/thrift/HeartbeatService.thrift b/gensrc/thrift/HeartbeatService.thrift index 8608853b29f81a..b47f84cb3bb0b5 100644 --- a/gensrc/thrift/HeartbeatService.thrift +++ b/gensrc/thrift/HeartbeatService.thrift @@ -64,6 +64,12 @@ struct TBackendInfo { 9: optional Types.TPort arrow_flight_sql_port 10: optional i64 be_mem // The physical memory available for use by BE. 11: optional bool supports_paimon_rust_reader + // The paimon-rust PK-vector (ANN) search protocol: paimon_split is empty and the + // split rides TPaimonFileDesc.vector_payload, which an older rust-capable BE + // rejects as "missing paimon_split". Set only by BEs that implement the vector + // protocol, so a mixed-version cluster keeps ordinary rust reads working and + // only declines the vector rewrite against older BEs. Field 12 is free on master. + 12: optional bool supports_paimon_rust_vector_reader // For cloud 1000: optional i64 fragment_executing_count 1001: optional i64 fragment_last_active_time diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 1488f75db78787..d7ce2d9414a323 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -360,6 +360,27 @@ enum TPaimonReaderType { PAIMON_RUST = 3, } +// Payload kind for Paimon primary-key vector (ANN) search. +// v1 only uses BUCKET_SPLIT_BYTES; v2 (delayed materialization) will add +// MATERIALIZE_OPAQUE_BYTES. Kept as an enum from v1 so the wire protocol is +// shared between v1 and v2 and does not need to change again. +enum TPaimonVectorPayloadType { + // v1 / v2 candidate stage: serialized BucketVectorSearchSplit + // (magic "PKVSPLIT", version 1, big-endian; decoded by paimon-rust + // paimon_bucket_vector_search_split_deserialize). + BUCKET_SPLIT_BYTES = 0, + // v2 materialize stage: opaque bytes produced by the merge/rerank step. + MATERIALIZE_OPAQUE_BYTES = 1, +} + +// Per-split vector payload carried on TPaimonFileDesc. The bytes are the raw +// BucketVectorSearchSplit.serialize form (NOT base64, NOT a Java +// ObjectOutputStream envelope) whose meaning is determined by payload_type. +struct TPaimonVectorPayload { + 1: required TPaimonVectorPayloadType payload_type; + 2: required binary bytes; +} + struct TPaimonFileDesc { 1: optional string paimon_split 2: optional string paimon_column_names @@ -385,6 +406,13 @@ struct TPaimonFileDesc { // paimon-rust: non-default branch name. Unset means main branch (matches // upstream paimon commit 742da63: null-if-DEFAULT_MAIN_BRANCH). 19: optional string paimon_branch + // Primary-key vector (ANN) search payload for this split. When set, BE + // routes to the paimon-rust vector-search read path instead of the normal + // split read; paimon_split is set but empty in that case. + // Field 21: id 20 is reserved as `original_file_path` on master and must not + // be reused here — a mixed-version cluster skips mismatched wire types + // instead of receiving either field. + 21: optional TPaimonVectorPayload vector_payload; } struct TTrinoConnectorFileDesc { @@ -560,6 +588,15 @@ struct TVectorSearchOptions { 5: optional i32 query_parallelism } +// Paimon-only vector search tuning. Mirrors TVectorSearchOptions: the logical query stays +// in TVectorSearchParams, provider-specific knobs live here so TExternalSearchRequest stays +// provider-neutral. The options map is passed straight through to the paimon-rust vector +// search builder (refine_factor, ivf.nprobe, and the ".metric" key that matches +// Paimon's algorithm options). +struct TPaimonVectorSearchOptions { + 1: optional map options +} + // The active union field identifies the logical search kind. A future hybrid field can contain both // vector and full-text subqueries plus its fusion parameters without changing either existing field. union TExternalSearchQuery { @@ -568,7 +605,8 @@ union TExternalSearchQuery { } // A provider-independent logical search request. Physical target information remains in the -// provider FileDesc (for example, dataset_uri/version/fragment_ids in TLanceFileDesc). +// provider FileDesc (for example, dataset_uri/version/fragment_ids in TLanceFileDesc, or +// vector_payload in TPaimonFileDesc). struct TExternalSearchRequest { 1: optional i32 schema_version = 1 2: optional TExternalSearchQuery search_query @@ -577,6 +615,9 @@ struct TExternalSearchRequest { // Applies only to search_filter. Defaults to false (postfilter); Doris WHERE predicates remain // residuals regardless of this setting. 5: optional bool prefilter + // Paimon-specific vector search tuning. Field 4 stays Lance-neutral, so Paimon's + // string-map knobs travel here instead; unset for Lance requests. + 6: optional TPaimonVectorSearchOptions paimon_options } // A catalog/S3 range reads fragments from a fixed snapshot. A local TVF range uses version zero @@ -728,6 +769,14 @@ struct TFileScanRangeParams { 39: optional i32 parquet_timestamp_semantics_version // Hybrid Paimon/Hudi scans keep FORMAT_JNI while individual ranges can be native Parquet. 40: optional bool contains_native_parquet + // Provider-independent external (vector / full-text) search request, set for a Paimon + // primary-key vector (ANN) top-N pushed down into the scan. Set at ScanNode level + // (shared by all splits); the per-split physical payload stays in + // TPaimonFileDesc.vector_payload. Lance requests use lance_scan_params instead. + // Field 42: id 41 is reserved as `fluss_properties` on master and must not be + // reused here — a mixed-version cluster skips mismatched wire types instead + // of receiving either field. + 42: optional TExternalSearchRequest external_search_request } struct TFileRangeDesc { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy index 897377b8e9dda2..72b78a36c5d716 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ResultUtils.groovy @@ -23,12 +23,14 @@ import org.junit.Assert import java.math.BigDecimal import java.math.BigInteger import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets import java.sql.Blob import java.sql.Clob import java.sql.Struct import java.sql.Time import java.sql.Timestamp import java.time.temporal.TemporalAccessor +import java.util.regex.Matcher import java.util.regex.Pattern @CompileStatic @@ -45,6 +47,9 @@ class ResultUtils { private static final Pattern DATETIME_PATTERN = Pattern.compile( "(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2})(?::(\\d{2})(?:\\.(\\d{1,9}))?)?") private static final Pattern UNICODE_ESCAPE_PATTERN = Pattern.compile("[0-9a-fA-F]{4}") + /** A "0x" token as Doris renders a VARBINARY nested in arrays/maps/structs. */ + private static final Pattern NESTED_BINARY_TOKEN = Pattern.compile( + "(?i)0x(?:[0-9a-f]{2})*(?![0-9a-fA-F])") static List> normalizeRows(List> rows) { List> normalizedRows = new ArrayList<>() @@ -59,8 +64,8 @@ class ResultUtils { } static void assertSparkDorisResultEquals(List> sparkRows, List> dorisRows) { - List> normalizedSparkRows = normalizeRows(sparkRows) - List> normalizedDorisRows = normalizeRows(dorisRows) + List> normalizedSparkRows = decodeNestedBinaryTokens(normalizeRows(sparkRows)) + List> normalizedDorisRows = decodeNestedBinaryTokens(normalizeRows(dorisRows)) Assert.assertEquals("Spark and Doris result row count mismatch", normalizedSparkRows.size(), normalizedDorisRows.size()) @@ -84,6 +89,87 @@ class ResultUtils { } } + /** + * Brings nested binary values rendered by the two engines to a common form. + * + *

Doris renders a VARBINARY nested in arrays/maps/structs as a + * "0x"-prefixed hex token ("[0x616C..., null]") while Spark's JDBC returns + * the raw bytes ("[alice-array-bin, null]"), so the same value compares + * unequal across engines. Decoding the tokens on both rows applies the same + * substitution to both sides — a no-op on the raw rendering — so equal stays + * equal, and scalar binary columns are unaffected: they arrive as byte[] + * and normalizeValue already hex-encodes them symmetrically. + */ + static List> decodeNestedBinaryTokens(List> rows) { + List> decodedRows = new ArrayList<>(rows.size()) + for (List row : rows) { + List decodedRow = new ArrayList<>(row.size()) + for (Object value : row) { + decodedRow.add(decodeNestedBinaryTokens(value)) + } + decodedRows.add(decodedRow) + } + return decodedRows + } + + static Object decodeNestedBinaryTokens(Object value) { + if (value instanceof String) { + String text = (String) value + Matcher matcher = NESTED_BINARY_TOKEN.matcher(text) + StringBuffer sb = new StringBuffer() + while (matcher.find()) { + String hex = matcher.group().substring(2) + String decoded = decodeHexString(hex) + if (decoded == null) { + // The bytes are not valid UTF-8: any String rendering would + // go through a lossy replacement character, collapsing + // distinct values (the single bytes 0x80 and 0x81 both + // become U+FFFD) and making unequal results compare equal. + // Keep the lossless 0x-hex token instead; a rendering that + // cannot round-trip then fails loudly rather than passing. + continue + } + matcher.appendReplacement(sb, Matcher.quoteReplacement(decoded)) + } + matcher.appendTail(sb) + return sb.toString() + } + if (value instanceof List) { + List decoded = new ArrayList<>(((List) value).size()) + for (Object item : (List) value) { + decoded.add(decodeNestedBinaryTokens(item)) + } + return decoded + } + if (value instanceof Map) { + Map decoded = new LinkedHashMap<>() + for (Map.Entry entry : ((Map) value).entrySet()) { + decoded.put(decodeNestedBinaryTokens(entry.getKey()), + decodeNestedBinaryTokens(entry.getValue())) + } + return decoded + } + return value + } + + /** + * Decodes a hex token to the String the other engine renders for the same + * bytes, or returns null when the bytes are not valid UTF-8: decoding then + * could only go through a lossy replacement character, and the caller keeps + * the lossless 0x-hex form instead. + */ + private static String decodeHexString(String hex) { + byte[] bytes = new byte[hex.length().intdiv(2)] + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16) + } + String decoded = new String(bytes, StandardCharsets.UTF_8) + if (!Arrays.equals(decoded.getBytes(StandardCharsets.UTF_8), bytes)) { + return null + } + return decoded + } + static Object normalizeValue(Object value) { if (value == null) { return null diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy new file mode 100644 index 00000000000000..d1716ebb802657 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_pk_vector_ann.groovy @@ -0,0 +1,395 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// End-to-end regression for the Paimon PK-vector (ANN) search path: the +// PushDownVectorTopNIntoPaimonScan rewrite, the shared rust eligibility gate +// and the BE paimon-rust vector reader, compared against the exact brute-force +// answer the same SQL computes with the rust reader disabled. +// +// The fixture (database pk_vector_ann_db, tables ann_l2 / ann_ip) is created +// by the suite itself through the Spark ThriftServer's paimon catalog: 512 +// deterministic 16-dim vectors derived from the row id (unique: the generator +// is mod 521, a prime above the row count), an ivf-rq PK-vector index with +// nlist = nprobe = 4 — exhaustive recall, so ANN must equal brute force by +// construction. Doris DDL cannot build this fixture: Doris has no VECTOR +// column type in either direction of the type mapping, and the ivf-rq index +// build needs the vindex factory jars that only the Spark server carries +// (paimon-vector + paimon-vector-index-java; see regression.md). +// +// Float precision note: the ANN distances are computed in f32 by paimon-rust +// while the brute-force leg computes in Doris doubles, so near-tied neighbors +// can swap positions between the legs. The assertions therefore compare the +// top-1 exactly (it is the probe row itself, distance 0), the first three in +// order, and the whole top-N as a set. +import org.apache.doris.regression.action.ProfileAction + +suite("test_paimon_pk_vector_ann", "p0,external") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disabled paimon test") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_paimon_pk_vector_ann" + String database = "pk_vector_ann_db" + int rows = 512 + String dimension = "16" + + // Session overrides are captured for the finally block the way the sibling + // suites do. + def originalRustReader = sql("select @@enable_paimon_rust_reader")[0][0] + def originalV2 = sql("select @@enable_file_scanner_v2")[0][0] + def originalEnableProfile = sql("select @@enable_profile")[0][0] + + try { + sql "set enable_profile=true" + sql """DROP CATALOG IF EXISTS ${catalogName}""" + // The s3.region property is required: paimon-rust's S3 client rejects a + // missing region, while the JNI reader falls back to the SDK default. + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.region' = 'us-east-1', + 'use_path_style' = 'true' + )""" + + // ------------------------------------------------------------------ fixture + // Recreated on every run, so the suite is self-contained. The vector + // column is declared ARRAY + 'field.embedding.vector-dim': the 2.1 + // connector turns that into a paimon VECTOR column, which no + // Doris DDL can express. The write commit builds the ivf-rq index inline + // through the vindex factory jars loaded on the Spark server. + String vectorOptions = '{"ivf-rq.nlist":"4","ivf-rq.dimension":"' + dimension + '",' + + '"ivf-rq.rq.bits":"4","ivf-rq.train.sample-ratio":"1"}' + String generatedRows = """ + SELECT id, CAST(id % 7 AS INT), + transform(sequence(0, 15), + i -> CAST(((id * 17 + i * 13) % 521 - 260) AS FLOAT) / 260.0) + FROM (SELECT explode(sequence(1, ${rows})) AS id)""" + [false, true].each { innerProduct -> + def table = innerProduct ? "ann_ip" : "ann_l2" + def metric = innerProduct ? "inner_product" : "l2" + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${database}; + DROP TABLE IF EXISTS paimon.${database}.${table}; + CREATE TABLE paimon.${database}.${table} ( + id BIGINT, tag INT, embedding ARRAY + ) USING paimon TBLPROPERTIES ( + 'primary-key' = 'id', 'bucket' = '2', + 'merge-engine' = 'partial-update', + 'deletion-vectors.enabled' = 'true', + 'deletion-vectors.merge-on-read' = 'false', + 'changelog-producer' = 'none', + 'vector-field' = 'embedding', + 'field.embedding.vector-dim' = '${dimension}', + 'pk-vector.index.columns' = 'embedding', + 'fields.embedding.pk-vector.index.type' = 'ivf-rq', + 'fields.embedding.pk-vector.distance.metric' = '${metric}', + 'fields.embedding.pk-vector.index.options' = '${vectorOptions}', + 'ivf.nprobe' = '4' + ); + INSERT INTO paimon.${database}.${table} ${generatedRows}; + """ + } + sql """switch ${catalogName}""" + sql "use ${database}" + def fixtureCount = sql("select count(*) from ann_l2") + assertEquals(rows.toLong(), fixtureCount[0][0], "fixture must have ${rows} rows") + + // The probe vector is the embedding of id=7 itself, taken from the data: + // its top-1 neighbor must be id=7 with distance 0, and every leg below + // uses the same in-data vector so ANN and brute force are comparable + // without any golden values. + def probeRow = sql("select embedding from ann_l2 where id = 7") + String probe = probeRow[0][0].toString() + + def idsOf = { rowsResult -> rowsResult.collect { it[0] } } + def compareTopN = { String table, boolean innerProduct, int n -> + def fn = innerProduct ? "inner_product_approximate" : "l2_distance_approximate" + def orderClause = innerProduct ? "desc" : "" + sql "set enable_paimon_rust_reader=0" + def brute = idsOf(sql("""select id from ${table} + order by ${fn}(embedding, ${probe}) ${orderClause} limit ${n}""")) + sql "set enable_paimon_rust_reader=1" + def ann = idsOf(sql("""select id from ${table} + order by ${fn}(embedding, ${probe}) ${orderClause} limit ${n}""")) + assertEquals(brute.first(), ann.first(), "top-1 must match exactly") + assertEquals(brute.take(3), ann.take(3), + "top-3 order must match: brute=${brute} ann=${ann}") + assertEquals(brute.sort(), ann.sort(), + "top-${n} sets must match: brute=${brute} ann=${ann}") + } + + // ------------------------------------------------------------------ l2 + def profileAction = new ProfileAction(context) + compareTopN("ann_l2", false, 5) + assertEquals(7L, idsOf(sql("""select id from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 1"""))[0], + "the probe row itself must be top-1") + + // The ANN leg must actually have executed on the rust vector reader — + // otherwise the equality above could pass on a silent brute-force scan. + def annQuery = """select id from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""" + sql annQuery + def queryId = sql("select last_query_id()")[0][0] + def profile = profileAction.getProfile(queryId.toString(), ["PaimonRustReader"]) + assertTrue(profile.contains("PaimonRustReader"), + "ANN query must run through the paimon-rust vector reader") + + // Non-index-only projection: the raw vector column is still readable + // alongside the distance; tag = (id - 1) % 7 by the fixture generator. + def annTagged = sql("""select id, tag from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""") + annTagged.each { row -> + assertEquals(row[0] % 7, row[1], "tag must be id % 7") + } + + // Residual filter: the filter rides to BE's vector-search stage and + // must agree with the filtered brute force (as a set; see the header). + sql "set enable_paimon_rust_reader=0" + def bruteFiltered = idsOf(sql("""select id from ann_l2 where tag < 3 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annFiltered = idsOf(sql("""select id from ann_l2 where tag < 3 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteFiltered, annFiltered, "filtered ANN must match filtered brute force") + + // Rewrite disabled by the scanner switch: still correct, just slower. + sql "set enable_file_scanner_v2=0" + def noRewrite = idsOf(sql("""select id from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_file_scanner_v2=1" + sql "set enable_paimon_rust_reader=0" + def bruteTop5 = idsOf(sql("""select id from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteTop5, noRewrite, "without FileScannerV2 the query must brute-force") + + // ------------------------------------------------------------------ + // Bounded-search soundness (review feedback on the rewrite): + // A predicate that cannot ride into the BE vector search must not be + // answered from a top-(LIMIT) fetch, so the rewrite declines for these + // shapes and the query runs with the distance computed in Doris. Every + // leg is a differential against the rust-disabled brute force. + + // A predicate on the distance itself: the synthetic distance column is + // outside the paimon-rust predicate converter's field registry, and a + // residual applied after the bounded fetch would return the wrong rows + // (with candidate distances 0/2/3, `dis > 1 LIMIT 1` must return the + // distance-2 row, not an empty result). + def distanceLowerBound = { + sql "set enable_paimon_rust_reader=0" + def brute = idsOf(sql("""select id from ann_l2 + where l2_distance_approximate(embedding, ${probe}) > 1 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def ann = idsOf(sql("""select id from ann_l2 + where l2_distance_approximate(embedding, ${probe}) > 1 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(brute, ann, "a distance predicate must not be served by a bounded fetch") + } + distanceLowerBound() + + // An unconvertible data predicate (the ARRAY vector column is not + // a paimon-rust predicate type): same soundness rule. + sql "set enable_paimon_rust_reader=0" + def bruteArrayPred = idsOf(sql("""select id from ann_l2 + where embedding is not null + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annArrayPred = idsOf(sql("""select id from ann_l2 + where embedding is not null + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteArrayPred, annArrayPred, + "an unconvertible predicate must not be served by a bounded fetch") + + // A pinned file-creation-time read state: the ordinary path drops manifest + // entries older than the bound, but a PK-vector search plans its own splits + // and never consults the bound -- so the rewrite declines and both legs + // must agree on the file-visible rows. A bound of 1 keeps every file; a + // far-future bound keeps none. + sql "set enable_paimon_rust_reader=0" + def bruteOld = idsOf(sql("""select id from ann_l2@options( + 'scan.file-creation-time-millis'='1') + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + def bruteNone = idsOf(sql("""select count(*) from ann_l2@options( + 'scan.file-creation-time-millis'='1791821967062000')""")) + sql "set enable_paimon_rust_reader=1" + def annOld = idsOf(sql("""select id from ann_l2@options( + 'scan.file-creation-time-millis'='1') + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + def annNone = idsOf(sql("""select count(*) from ann_l2@options( + 'scan.file-creation-time-millis'='1791821967062000')""")) + assertEquals(bruteOld, annOld, "a file-creation-time bound must be honored on the vector path") + assertEquals(bruteNone, annNone, "a far-future file bound must exclude all files on both paths") + + // -------------------------------------------------------- more fixtures + // Two additional shapes the bounded rewrite must handle: a PK-vector table + // whose merge engine permits the index WITHOUT deletion vectors (first-row, + // allowed since paimon 2.1), and a partitioned PK-vector table (whose vector + // splits must carry the derived partition values or every scan fails with + // "Missing partition values"). + spark_paimon_multi """ + CREATE TABLE IF NOT EXISTS paimon.${database}.ann_l2_nodv ( + id BIGINT, tag INT, embedding ARRAY + ) USING paimon TBLPROPERTIES ( + 'primary-key' = 'id', 'bucket' = '2', + 'merge-engine' = 'first-row', + 'changelog-producer' = 'none', + 'vector-field' = 'embedding', + 'field.embedding.vector-dim' = '${dimension}', + 'pk-vector.index.columns' = 'embedding', + 'fields.embedding.pk-vector.index.type' = 'ivf-rq', + 'fields.embedding.pk-vector.distance.metric' = 'l2', + 'fields.embedding.pk-vector.index.options' = '${vectorOptions}', + 'ivf.nprobe' = '4' + ); + INSERT INTO paimon.${database}.ann_l2_nodv ${generatedRows}; + """ + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${database}; + DROP TABLE IF EXISTS paimon.${database}.ann_l2_part; + CREATE TABLE paimon.${database}.ann_l2_part ( + id BIGINT, dt STRING, tag INT, embedding ARRAY + ) USING paimon PARTITIONED BY (dt) TBLPROPERTIES ( + 'primary-key' = 'id', 'bucket' = '2', + 'merge-engine' = 'partial-update', + 'deletion-vectors.enabled' = 'true', + 'deletion-vectors.merge-on-read' = 'false', + 'changelog-producer' = 'none', + 'vector-field' = 'embedding', + 'field.embedding.vector-dim' = '${dimension}', + 'pk-vector.index.columns' = 'embedding', + 'fields.embedding.pk-vector.index.type' = 'ivf-rq', + 'fields.embedding.pk-vector.distance.metric' = 'l2', + 'fields.embedding.pk-vector.index.options' = '${vectorOptions}', + 'ivf.nprobe' = '4' + ); + INSERT INTO paimon.${database}.ann_l2_part + SELECT id, IF(id % 2 = 0, '2026-01-01', '2026-01-02'), CAST(id % 7 AS INT), + transform(sequence(0, 15), + i -> CAST(((id * 17 + i * 13) % 521 - 260) AS FLOAT) / 260.0) + FROM (SELECT explode(sequence(1, ${rows})) AS id); + """ + assertEquals(rows.toLong(), sql("select count(*) from ann_l2_nodv")[0][0]) + assertEquals(rows.toLong(), sql("select count(*) from ann_l2_part")[0][0]) + + // ------------------------------------------------ no-deletion-vector shape + // Unfiltered: paimon-rust's vector search has no data predicate, so the + // rewrite stays sound on the no-DV table. + sql "set enable_paimon_rust_reader=0" + def bruteNoDv = idsOf(sql("""select id from ann_l2_nodv + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + // Filtered: the rust PkVectorSearchParams rejects every data predicate + // unless deletion vectors are enabled without merge-on-read, so the rewrite + // must decline here and both legs agree through the ordinary path. + def bruteNoDvFilter = idsOf(sql("""select id from ann_l2_nodv where tag < 3 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annNoDv = idsOf(sql("""select id from ann_l2_nodv + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + def annNoDvFilter = idsOf(sql("""select id from ann_l2_nodv where tag < 3 + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteNoDv, annNoDv, "unfiltered ANN must match brute force on the no-DV table") + assertEquals(bruteNoDvFilter, annNoDvFilter, + "a data predicate on a no-deletion-vector table must not run as a bounded ANN") + + // ------------------------------------------------------------ partitioned + // The vector splits of a partitioned table must carry the derived partition + // values (setPaimonParams throws "Missing partition values" otherwise), and + // the ANN result must still match the ordinary brute-force answer. + sql "set enable_paimon_rust_reader=0" + def brutePart = idsOf(sql("""select id from ann_l2_part + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annPart = idsOf(sql("""select id from ann_l2_part + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(brutePart, annPart, "partitioned ANN must match brute force") + + // ---------------------------------------------- NULL query-vector element + // A NULL element inside the query vector must not become a zero coordinate: + // the ordinary function rejects it, and the rewrite must decline so both + // legs raise the same validation error. + String nullVector = "array(cast(null as float)" + + (1..(dimension.toInteger() - 1)).collect { ", 0.0" }.join() + ")" + def queryWithNulls = """select id from ann_l2 + order by l2_distance_approximate(embedding, ${nullVector}) limit 5""" + def expectValidationError = { String q -> + try { + sql q + return null + } catch (Throwable t) { + return t.getMessage() + } + } + sql "set enable_paimon_rust_reader=0" + def bruteError = expectValidationError(queryWithNulls) + sql "set enable_paimon_rust_reader=1" + def annError = expectValidationError(queryWithNulls) + assertNotNull("the ordinary path must reject a NULL query vector", bruteError) + assertEquals(bruteError, annError, + "the ANN path must reject a NULL query vector the same way, not search a zero vector") + + + // ------------------------------------------- non-movable project expression + // assert_true in the projection must keep evaluating below the TopN: the + // ordinary path raises for the rejected far rows (the fixture has id%7=0 + // tags), while a bounded fetch that truncates before the Project would + // silently succeed. The rewrite declines, so both legs raise the same error. + def queryWithAssertTrue = """select id, assert_true(tag > 0, 'bad tag') from ann_l2 + order by l2_distance_approximate(embedding, ${probe}) limit 5""" + sql "set enable_paimon_rust_reader=0" + def bruteAssertError = expectValidationError(queryWithAssertTrue) + sql "set enable_paimon_rust_reader=1" + def annAssertError = expectValidationError(queryWithAssertTrue) + assertNotNull("the ordinary path must reject a failing assert_true", bruteAssertError) + assertEquals(bruteAssertError, annAssertError, + "a non-movable projection must not be truncated by a bounded fetch") + + // ------------------------------------------- IN predicate with a NULL option + // tag IN (0, 1, NULL) passes the shape whitelist (every option is a + // literal) but the BE converter rejects the null option and with it the + // whole predicate, so the rewrite must decline: the IN stays in the + // residual and both legs must agree. + sql "set enable_paimon_rust_reader=0" + def bruteInNull = idsOf(sql("""select id from ann_l2 where tag in (0, 1, null) + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + sql "set enable_paimon_rust_reader=1" + def annInNull = idsOf(sql("""select id from ann_l2 where tag in (0, 1, null) + order by l2_distance_approximate(embedding, ${probe}) limit 5""")).sort() + assertEquals(bruteInNull, annInNull, "an IN with a NULL option must not run as a bounded ANN") + + // -------------------------------------------------------- inner product + compareTopN("ann_ip", true, 5) + } finally { + sql "set enable_paimon_rust_reader=${originalRustReader}" + sql "set enable_file_scanner_v2=${originalV2}" + sql "set enable_profile=${originalEnableProfile}" + try { + sql """DROP CATALOG IF EXISTS ${catalogName}""" + } catch (Throwable t) { + logger.info("drop catalog failed: " + t) + } + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy index 274bc1c67ab069..0f85825e9d783c 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy @@ -57,14 +57,19 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { } def snapshotCommitTimeMillis = { String tableName, String snapshotId -> - List> rows = spark_paimon """ - -- Paimon's TIMESTAMP_NTZ commit time is a UTC wall clock; convert it to the - -- Spark session zone before casting so unix_millis keeps the actual instant. - SELECT unix_millis(CAST( - convert_timezone('UTC', current_timezone(), commit_time) AS TIMESTAMP - )) - FROM paimon.${dbName}.`${tableName}\$snapshots` - WHERE snapshot_id = ${snapshotId} + // Read the commit time through Doris so the epoch reference matches the + // scan.creation-time-millis resolution (the earliest snapshot committed at + // or after the value) on the same clock: Doris' unix_timestamp keeps the + // fractional seconds, so * 1000 is the exact commit instant. The previous + // Spark-side convert_timezone + unix_millis conversion assumed the NTZ + // commit time is a UTC wall clock, which the 2.1 ThriftServer no longer + // holds — the value came out shifted by the session UTC offset (8h into + // the future) and every creation-time scan silently resolved to the latest + // snapshot. + List> rows = sql """ + select cast(unix_timestamp(commit_time) * 1000 as bigint) + from ${catalogName}.${dbName}.${tableName}\$snapshots + where snapshot_id = ${snapshotId} """ assertEquals(1, rows.size()) return rows[0][0].toString() diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_orc_ltz_encoding.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_orc_ltz_encoding.groovy index fc3f30477c50ac..c1b10059f7b25d 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_orc_ltz_encoding.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_orc_ltz_encoding.groovy @@ -55,7 +55,19 @@ suite("test_paimon_orc_ltz_encoding", "p0,external,paimon") { SHOW TBLPROPERTIES paimon.${dbName}.${tableName} ('${legacyKey}') """)) } - assertEncoding("t_modern", "false") + // The paimon 1.4 connector materialized unset property defaults into the stored + // table options, so t_modern (created without the legacy flag) read back as + // "false". The 2.1 connector no longer persists unset defaults, so the same + // query fails with "does not have property" — which still means the modern + // default. Anything else (a persisted "true", or any other error) must fail. + boolean modernDefault = false + try { + assertEncoding("t_modern", "false") + modernDefault = true + } catch (Throwable t) { + modernDefault = t.toString().contains("does not have property") + } + assertTrue(modernDefault, "t_modern must keep the modern (non-legacy) encoding default") sql """CREATE TABLE t_legacy (event_time TIMESTAMPTZ(6)) ENGINE=paimon PROPERTIES ('file.format' = 'orc', '${legacyKey}' = 'true')""" sql """CREATE TABLE IF NOT EXISTS t_legacy (event_time TIMESTAMPTZ(6)) ENGINE=paimon""" diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy index bfdeffa376bbe6..715bcd178c2c4f 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy @@ -447,7 +447,7 @@ suite("test_paimon_write_transaction", "p0,external,paimon") { sql """INSERT INTO t_failed_write VALUES (2, 'accepted_before_error'), (3, repeat('x', 1048576))""" - exception "The record exceeds the maximum size of a sort buffer" + exception "The record exceeds the maximum size of a" } qt_txn_failed_write_after """SELECT id, payload FROM t_failed_write ORDER BY id""" qt_txn_failed_snapshot_after """SELECT COUNT(*) FROM t_failed_write\$snapshots"""