diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 13f6ebe986c887..fcdd403a43ba29 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -894,6 +894,8 @@ else() "-framework Foundation" "-framework SystemConfiguration" "-framework Security" + # liblance_c.a pulls in the Rust `sysinfo` crate, whose macOS backend calls IOKit. + "-framework IOKit" ) if (USE_JEMALLOC OR (NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG" AND NOT CMAKE_BUILD_TYPE STREQUAL "RELEASE")) set(DORIS_LINK_LIBS diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 4c9957dfdfa173..55f26dbc28efd3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1232,6 +1232,26 @@ DEFINE_Validator(variant_max_json_key_length, DEFINE_Validator(variant_storage_parse_mode, [](const int config) -> bool { return config >= 0 && config <= 2; }); +// Lance uses one BE-wide session so metadata/index caches and the optional Foyer data-file cache +// can be shared by all Lance dataset readers. +DEFINE_Int64(lance_index_cache_size_bytes, "10737418240"); // 10 GiB +DEFINE_Int64(lance_metadata_cache_size_bytes, "1073741824"); // 1GB +DEFINE_Bool(enable_lance_data_cache, "true"); +DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache"); +DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB +DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB + +// I/O buffering budget per Lance scanner, not a cap on its total memory usage. +// Runtime changes apply to newly created scanners. +DEFINE_mInt64(lance_io_buffer_size_bytes, "2147483648"); // 2 GiB +DEFINE_Validator(lance_io_buffer_size_bytes, [](int64_t value) { return value > 0; }); + +// Read-ahead limits per Lance scanner. Runtime changes apply to newly created scanners. +DEFINE_mInt32(lance_batch_readahead, "5"); +DEFINE_Validator(lance_batch_readahead, [](int32_t value) { return value > 0; }); +DEFINE_mInt32(lance_fragment_readahead, "5"); +DEFINE_Validator(lance_fragment_readahead, [](int32_t value) { return value > 0; }); + // block file cache DEFINE_Bool(enable_file_cache, "true"); // ATTENTION: For test only. Keep this enabled in production. diff --git a/be/src/common/config.h b/be/src/common/config.h index 62becc90f28a1e..cd72cc2f5fb5b5 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1286,6 +1286,21 @@ DECLARE_Bool(enable_debug_points); DECLARE_Int32(pipeline_executor_size); DECLARE_Int32(blocking_pipeline_executor_size); +// Lance shared session and optional Foyer data-file cache. +DECLARE_Int64(lance_index_cache_size_bytes); +DECLARE_Int64(lance_metadata_cache_size_bytes); +DECLARE_Bool(enable_lance_data_cache); +DECLARE_String(lance_data_cache_path); +DECLARE_Int64(lance_data_cache_disk_capacity_bytes); +DECLARE_Int64(lance_data_cache_read_block_size_bytes); + +// I/O buffering budget per Lance scanner, applied when a new scanner is created. +DECLARE_mInt64(lance_io_buffer_size_bytes); + +// Read-ahead limits per Lance scanner, applied when a new scanner is created. +DECLARE_mInt32(lance_batch_readahead); +DECLARE_mInt32(lance_fragment_readahead); + // block file cache DECLARE_Bool(enable_file_cache); DECLARE_mBool(enable_file_cache_write_from_s3_file_writer); diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp index 88b6e473a2be9c..1b0a50ef8cd0fd 100644 --- a/be/src/core/data_type/data_type.cpp +++ b/be/src/core/data_type/data_type.cpp @@ -84,7 +84,9 @@ void IDataType::to_protobuf(PTypeDesc* ptype) const { auto node = ptype->add_types(); node->set_type(TTypeNodeType::SCALAR); auto scalar_type = node->mutable_scalar_type(); - scalar_type->set_type(doris::to_thrift(get_primitive_type())); + // NULL uses UInt8 internally; preserve its logical type even inside complex schemas. + scalar_type->set_type(is_null_literal() ? TPrimitiveType::NULL_TYPE + : doris::to_thrift(get_primitive_type())); to_protobuf(ptype, node, scalar_type); } diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp b/be/src/core/data_type_serde/data_type_datev2_serde.cpp index b7021af6f01503..80b6ebd5e8a575 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp @@ -277,11 +277,27 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow: const auto* base_ptr = reinterpret_cast(concrete_array->raw_values()); const size_t element_size = sizeof(int32_t); for (auto value_i = start; value_i < end; ++value_i) { + // A null slot has no calendar value and its payload may be outside the DATE range. + if (concrete_array->IsNull(value_i)) { + col_data.emplace_back(DateV2Value()); + continue; + } const uint8_t* raw_byte_ptr = base_ptr + value_i * element_size; auto date_value = unaligned_load(raw_byte_ptr); + // Arrow allows any int32 day, Doris DATE only day numbers 1..DATE_MAX_DAYNR + // (0000-01-01..9999-12-31). For example, 2932897 is 10000-01-01: its day number + // 3652425 makes get_date_from_daynr() fail, and ignoring that result used to append + // a non-null 0000-00-00. Add in 64 bits so INT32_MAX cannot overflow, and reject + // the value like the DATE64 branch above. + const int64_t daynr = static_cast(date_value) + date_threshold; DateV2Value v; - v.get_date_from_daynr(date_value + date_threshold); + if (daynr <= 0 || daynr > DATE_MAX_DAYNR || + !v.get_date_from_daynr(static_cast(daynr))) { + return Status::InvalidArgument( + "Arrow Date32 value is outside the Doris DATE range: row={}, days={}", + value_i, date_value); + } col_data.emplace_back(v); } } else { diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index e354d423f0b2f6..4c4d0472f48720 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -410,6 +410,18 @@ Status DataTypeNullableSerDe::read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, int64_t end, const cctz::time_zone& ctz) const { + if (arrow_array->type_id() == arrow::Type::NA) { + // Arrow Null has neither a validity bitmap nor values, even when sliced. Avoid the + // physical SerDe for Doris's UInt8 placeholder and keep nested values and nulls aligned. + if (arrow_array->offset() < 0 || start < 0 || end < start || end > arrow_array->length()) { + return Status::InvalidArgument( + "Invalid Arrow Null read range: start={}, end={}, " + "length={}, offset={}", + start, end, arrow_array->length(), arrow_array->offset()); + } + column.insert_many_defaults(end - start); + return Status::OK(); + } if (config::enable_arrow_input_validation) { check_arrow_array_range(*arrow_array, start, end); check_arrow_validity_bitmap(*arrow_array); diff --git a/be/src/exec/operator/file_scan_operator.cpp b/be/src/exec/operator/file_scan_operator.cpp index e439f308ef4f16..871b62e700d960 100644 --- a/be/src/exec/operator/file_scan_operator.cpp +++ b/be/src/exec/operator/file_scan_operator.cpp @@ -197,9 +197,12 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o // fluss scan carries that same format -- log ranges and wrapped lake splits alike -- and is // dispatched inside FlussHybridReader, which only FileScannerV2 builds. Loads are left alone for // the same reason as adbc: there is no fluss load path. - if (!is_load && scan_params.__isset.table_format_params && - (scan_params.table_format_params.table_format_type == "adbc" || - scan_params.table_format_params.table_format_type == "fluss")) { + // Lance has no v1 reader either. Its scans carry FORMAT_LANCE at scan level, whether they come + // from a Lance catalog or from a file TVF, and there is no Lance load path. + if (!is_load && (scan_params.format_type == TFileFormatType::FORMAT_LANCE || + (scan_params.__isset.table_format_params && + (scan_params.table_format_params.table_format_type == "adbc" || + scan_params.table_format_params.table_format_type == "fluss")))) { return true; } const bool is_transactional_hive = @@ -224,8 +227,7 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o // after scanner construction. Keep versioned Iceberg scans on V2 without eagerly listing // every file. return scanner_v2_requested && !is_load && - scan_params.format_type != TFileFormatType::FORMAT_ES_HTTP && - scan_params.format_type != TFileFormatType::FORMAT_LANCE && !is_transactional_hive; + scan_params.format_type != TFileFormatType::FORMAT_ES_HTTP && !is_transactional_hive; } Status FileScanLocalState::_init_scanners(std::list* scanners) { diff --git a/be/src/exec/operator/materialization_opertor.cpp b/be/src/exec/operator/materialization_opertor.cpp index f8871c533c5c2d..60c31ecb019777 100644 --- a/be/src/exec/operator/materialization_opertor.cpp +++ b/be/src/exec/operator/materialization_opertor.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -72,6 +73,57 @@ constexpr const char* TOPN_LAZY_MAT_PHASE2_PER_BACKEND_REMOTE_IO_TIME = constexpr const char* TOPN_LAZY_MAT_PHASE2_PER_BACKEND_WRITE_CACHE_IO_TIME = "TopNLazyMaterializationSecondPhasePerBackendWriteCacheIOTime"; +struct MaterializationRowLocation { + ROW_VERSION version = ROW_VERSION::FILE_LOCAL_ROW_ID; + int64_t backend_id = 0; + uint32_t file_id = 0; + uint64_t row_id = 0; +}; + +Status decode_global_row_location_v2(const StringRef& encoded, ROW_VERSION version, + MaterializationRowLocation* decoded) { + if (encoded.size != sizeof(GlobalRowLoacationV2)) { + return Status::InternalError( + "invalid global row location size for version {}: actual={}, expected={}", + static_cast(version), encoded.size, sizeof(GlobalRowLoacationV2)); + } + GlobalRowLoacationV2 location(GlobalRowLoacationV2::VERSION, 0, 0, 0); + std::memcpy(&location, encoded.data, sizeof(location)); + decoded->version = version; + decoded->backend_id = location.backend_id; + switch (version) { + case ROW_VERSION::FILE_LOCAL_ROW_ID: + decoded->file_id = location.file_local.file_id; + decoded->row_id = location.file_local.row_id; + return Status::OK(); + case ROW_VERSION::LANCE_DATASET_ROW_ID: + decoded->file_id = location.lance_file_id; + decoded->row_id = location.lance_row_id; + return Status::OK(); + } + return Status::NotSupported("unsupported V2 global row location version: {}", + static_cast(version)); +} + +Status decode_materialization_row_location(const StringRef& encoded, + MaterializationRowLocation* decoded) { + if (encoded.size < sizeof(uint8_t)) { + return Status::InternalError("global row location is empty"); + } + + const auto version_value = static_cast(encoded.data[0]); + const auto version = static_cast(version_value); + // Keep size validation inside each version-family decoder. A future version can therefore + // use a different encoded size without being rejected by the current 24-byte V2 contract. + switch (version) { + case ROW_VERSION::FILE_LOCAL_ROW_ID: + case ROW_VERSION::LANCE_DATASET_ROW_ID: + return decode_global_row_location_v2(encoded, version, decoded); + } + return Status::NotSupported("unsupported global row location version: {}, encoded_size={}", + static_cast(version_value), encoded.size); +} + void update_counter(RuntimeProfile* profile, const std::string& name, TUnit::type unit, int64_t value) { COUNTER_UPDATE(ADD_COUNTER_WITH_LEVEL(profile, name, unit, 2), value); @@ -403,16 +455,20 @@ Status MaterializationSharedState::validate_rpc_results(int node_id) { void MaterializationSharedState::_update_profile_info(int64_t backend_id, RuntimeProfile* response_profile) { + DORIS_CHECK(response_profile != nullptr); if (!backend_profile_info_string.contains(backend_id)) { backend_profile_info_string.emplace(backend_id, std::map {}); } auto& info_map = backend_profile_info_string[backend_id]; - auto update_profile_info_key = [&](const std::string& info_key) { + auto update_profile_info_key = [&](const std::string& info_key, bool warn_if_missing = true) { const auto* info_value = response_profile->get_info_string(info_key); if (info_value == nullptr) [[unlikely]] { - LOG(WARNING) << "Get row id fetch rpc profile success, but no info key :" << info_key; + if (warn_if_missing) { + LOG(WARNING) << "Get row id fetch rpc profile success, but no info key :" + << info_key; + } return; } if (!info_map.contains(info_key)) { @@ -427,6 +483,13 @@ void MaterializationSharedState::_update_profile_info(int64_t backend_id, update_profile_info_key(RowIdStorageReader::FileReadLinesProfile); update_profile_info_key(FileScanner::FileReadBytesProfile); update_profile_info_key(FileScanner::FileReadTimeProfile); + update_profile_info_key(RowIdStorageReader::LanceDatasetOpenTimeProfile, false); + update_profile_info_key(RowIdStorageReader::LanceRowIdTakeReadTimeProfile, false); + update_profile_info_key(RowIdStorageReader::LanceArrowToDorisBlockTimeProfile, false); + update_profile_info_key(RowIdStorageReader::LanceRowIdFetchTotalTimeProfile, false); + for (const auto& [name, unit] : RowIdStorageReader::LanceFetchCountersProfile) { + update_profile_info_key(name, false); + } } Status MaterializationSharedState::create_muiltget_result(const Columns& columns, bool child_eos) { @@ -451,19 +514,28 @@ Status MaterializationSharedState::create_muiltget_result(const Columns& columns for (int j = 0; j < rows; ++j) { if (!null_map || !null_map[j]) { - DCHECK(column_rowid->get_data_at(j).size == sizeof(GlobalRowLoacationV2)); - GlobalRowLoacationV2 row_location = - *((GlobalRowLoacationV2*)column_rowid->get_data_at(j).data); + MaterializationRowLocation row_location; + RETURN_IF_ERROR(decode_materialization_row_location(column_rowid->get_data_at(j), + &row_location)); auto rpc_struct = rpc_struct_map.find(row_location.backend_id); if (UNLIKELY(rpc_struct == rpc_struct_map.end())) { return Status::InternalError( "MaterializationSinkOperatorX failed to find rpc_struct, backend_id={}", row_location.backend_id); } - rpc_struct->second.request.mutable_request_block_descs(i)->add_row_id( - row_location.row_id); - rpc_struct->second.request.mutable_request_block_descs(i)->add_file_id( - row_location.file_id); + auto* request_block_desc = + rpc_struct->second.request.mutable_request_block_descs(i); + const auto row_location_version = static_cast(row_location.version); + if (request_block_desc->row_id_size() == 0) { + request_block_desc->set_row_location_version(row_location_version); + } else if (request_block_desc->row_location_version() != row_location_version) { + return Status::InternalError( + "mixed row location versions in one materialization request: " + "actual={}, expected={}", + row_location_version, request_block_desc->row_location_version()); + } + request_block_desc->add_row_id(row_location.row_id); + request_block_desc->add_file_id(row_location.file_id); block_order[j] = row_location.backend_id; // Count rows per backend @@ -593,8 +665,7 @@ Status MaterializationOperator::pull(RuntimeState* state, Block* output_block, b local_state._materialization_state.backend_profile_info_string) { auto* child_profile = local_state.operator_profile()->create_child( "RowIDFetcher: BackendId:" + std::to_string(backend_id)); - for (const auto& [info_key, info_value] : - local_state._materialization_state.backend_profile_info_string[backend_id]) { + for (const auto& [info_key, info_value] : child_info) { child_profile->add_info_string(info_key, "{" + fmt::to_string(info_value) + "}"); } local_state.operator_profile()->add_child(child_profile, true); diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index 3f9c0efc82ebd9..6cac9778d0ec95 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -47,6 +48,7 @@ #include "exec/scan/file_scanner_v2.h" #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_reader.h" +#include "format_v2/table/lance_reader.h" #include "io/io_common.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" // ExecEnv @@ -61,6 +63,7 @@ #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" #include "util/jsonb/serialize.h" +#include "util/pretty_printer.h" namespace doris { @@ -244,6 +247,27 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequestV2& request, const auto& request_block_desc = request.request_block_descs(i); PMultiGetBlockV2* pblock = response->add_blocks(); if (request_block_desc.row_id_size() >= 1) { + // Every row is addressed by the pair (file_id(j), row_id(j)). A sender that + // appended 2 row IDs but 1 file ID would make the per-row lookups below read + // file_id(1), which does not exist. + if (request_block_desc.file_id_size() != request_block_desc.row_id_size()) { + return Status::InvalidArgument( + "Row-id fetch has mismatched file_id and row_id counts: " + "file_ids={}, row_ids={}", + request_block_desc.file_id_size(), request_block_desc.row_id_size()); + } + // A version this binary does not know (for example 2 from a newer sender) may use + // a different row ID meaning; reading it as FILE_LOCAL_ROW_ID would return + // unrelated rows instead of failing. + const auto row_location_version = request_block_desc.row_location_version(); + switch (row_location_version) { + case static_cast(ROW_VERSION::FILE_LOCAL_ROW_ID): + case static_cast(ROW_VERSION::LANCE_DATASET_ROW_ID): + break; + default: + return Status::NotSupported("unsupported row location version in fetch: {}", + row_location_version); + } // Since this block belongs to the same table, we only need to take the first type for judgment. auto first_file_id = request_block_desc.file_id(0); auto first_file_mapping = id_file_map->get_file_mapping(first_file_id); @@ -253,6 +277,14 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequestV2& request, BackendOptions::get_localhost(), print_id(request.query_id()), first_file_id); } + // Only a Lance scan produces LANCE_DATASET_ROW_ID, and Lance mappings are external. + // An internal segment never has a dataset row ID to take. + if (first_file_mapping->type != FileMappingType::EXTERNAL && + row_location_version == + static_cast(ROW_VERSION::LANCE_DATASET_ROW_ID)) { + return Status::InvalidArgument( + "Lance dataset row IDs require an external file mapping"); + } file_type_counts[first_file_mapping->type] += request_block_desc.row_id_size(); // prepare slots to build block @@ -359,6 +391,13 @@ Status RowIdStorageReader::read_batch_doris_format_row( std::vector> row_id_block_idx(request_block_desc.row_id_size()); for (int j = 0; j < request_block_desc.row_id_size(); ++j) { auto file_id = request_block_desc.file_id(j); + // The wire row ID is uint64 for every row location version. An internal segment ordinal + // is a FILE_LOCAL_ROW_ID that the sender zero-extended from uint32, so a value such as + // 2^32 can only come from a malformed request and must not be truncated to row 0. + const auto row_id = request_block_desc.row_id(j); + if (row_id > std::numeric_limits::max()) { + return Status::InvalidArgument("internal row id exceeds uint32 range: {}", row_id); + } auto file_mapping = id_file_map->get_file_mapping(file_id); if (!file_mapping) { return Status::InternalError( @@ -377,8 +416,8 @@ Status RowIdStorageReader::read_batch_doris_format_row( scan_batches.back().file_mapping = file_mapping; } // Record (row_id, original_request_index) for later sorting and scattering. - scan_batches[it->second].row_ids_with_positions.emplace_back(request_block_desc.row_id(j), - j); + scan_batches[it->second].row_ids_with_positions.emplace_back( + static_cast(row_id), j); } // Phase 2: For each segment, sort row_ids ascending (required by ColumnIterator), @@ -422,6 +461,17 @@ const std::string RowIdStorageReader::ScannersRunningTimeProfile = "ScannersRunn const std::string RowIdStorageReader::InitReaderAvgTimeProfile = "InitReaderAvgTime"; const std::string RowIdStorageReader::GetBlockAvgTimeProfile = "GetBlockAvgTime"; const std::string RowIdStorageReader::FileReadLinesProfile = "FileReadLines"; +const std::string RowIdStorageReader::LanceDatasetOpenTimeProfile = "LanceDatasetOpenTime"; +const std::string RowIdStorageReader::LanceRowIdTakeReadTimeProfile = "LanceRowIdTakeReadTime"; +const std::string RowIdStorageReader::LanceArrowToDorisBlockTimeProfile = + "LanceArrowToDorisBlockTime"; +const std::string RowIdStorageReader::LanceRowIdFetchTotalTimeProfile = "LanceRowIdFetchTotalTime"; +const std::map RowIdStorageReader::LanceFetchCountersProfile = { + {"LanceRowIdFetchRows", TUnit::UNIT}, + {"LanceRowIdFetchCalls", TUnit::UNIT}, + {"LanceDataCacheBytesReadFromCache", TUnit::BYTES}, + {"LanceDataCacheBytesReadFromRemote", TUnit::BYTES}, +}; const std::string RowIdStorageReader::TopNLazyMaterializationSecondPhaseLocalIOCount = "TopNLazyMaterializationSecondPhaseLocalIOCount"; const std::string RowIdStorageReader::TopNLazyMaterializationSecondPhaseLocalIOBytes = @@ -467,8 +517,66 @@ TFileRangeDesc RowIdStorageReader::build_external_fetch_range(const TFileRangeDe return range; } +Status RowIdStorageReader::read_lance_rows_by_row_ids( + const TFileRangeDesc& scan_range_desc, const std::vector& row_ids, + const std::vector& slots, RuntimeState* runtime_state, + RuntimeProfile* runtime_profile, const TFileScanRangeParams& scan_params, Block* block, + RowIdStorageReader::ExternalFetchStatistics* fetch_statistics) { + // Unlike Parquet/ORC, a Lance row ID is a native uint64 ID in a fixed dataset snapshot, + // rather than a row ordinal interpreted by a reader for one physical file range. Phase-two + // fetch therefore opens that dataset snapshot and uses dataset-level take_rows directly. It + // does not create a FileScanner or rescan the fragment split that produced the row ID. + std::vector projected_columns; + projected_columns.reserve(slots.size()); + for (const auto& slot : slots) { + format::ColumnDefinition column; + column.identifier = Field::create_field(slot.col_name()); + column.name = slot.col_name(); + column.type = slot.type(); + projected_columns.emplace_back(std::move(column)); + } + + format::lance::LanceTableReader reader; + auto lance_scan_params = scan_params; + RETURN_IF_ERROR(scope_timer_run( + [&]() { + return reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = format::FileFormat::LANCE, + .scan_params = &lance_scan_params, + .io_ctx = nullptr, + .runtime_state = runtime_state, + .scanner_profile = runtime_profile, + }); + }, + &fetch_statistics->init_reader_ms)); + RETURN_IF_ERROR(scope_timer_run( + [&]() { return reader.read_by_row_ids(scan_range_desc, row_ids, block); }, + &fetch_statistics->get_block_ms)); + RETURN_IF_ERROR(reader.close()); + + const auto collect_lance_fetch_time = [&](const std::string& timer_name) { + if (const auto* counter = runtime_profile->get_counter(timer_name); counter != nullptr) { + fetch_statistics->lance_fetch_times_ns.emplace(timer_name, counter->value()); + } + }; + collect_lance_fetch_time(LanceDatasetOpenTimeProfile); + collect_lance_fetch_time(LanceRowIdTakeReadTimeProfile); + collect_lance_fetch_time(LanceArrowToDorisBlockTimeProfile); + collect_lance_fetch_time(LanceRowIdFetchTotalTimeProfile); + // close() publishes dataset-handle cache statistics; collect after it and preserve the + // units across the RPC instead of formatting byte/count values as nanoseconds. + for (const auto& [name, unit] : LanceFetchCountersProfile) { + if (const auto* counter = runtime_profile->get_counter(name); counter != nullptr) { + fetch_statistics->lance_fetch_counters.emplace(name, counter->value()); + } + } + return Status::OK(); +} + Status RowIdStorageReader::read_external_row_from_file_mapping( - size_t idx, const std::multimap& row_ids, + size_t idx, const std::multimap& row_ids, const std::shared_ptr& file_mapping, const std::vector& scan_slots, const TUniqueId& query_id, const std::shared_ptr& runtime_state, std::vector& scan_blocks, @@ -485,8 +593,8 @@ Status RowIdStorageReader::read_external_row_from_file_mapping( // the caller, so the status is always published before the waiter is woken. Defer defer([&] { semaphore.release(); }); - std::list read_ids; - //Generate an ordered list with the help of the orderliness of the map. + std::vector read_ids; + // Generate an ordered, deduplicated list with the help of the multimap ordering. for (const auto& [row_id, result_block_idx] : row_ids) { if (read_ids.empty() || read_ids.back() != row_id) { read_ids.emplace_back(row_id); @@ -501,15 +609,31 @@ Status RowIdStorageReader::read_external_row_from_file_mapping( std::unique_ptr sub_runtime_profile = std::make_unique("ExternalRowIDFetcher"); - { + const auto format_type = scan_range_desc.__isset.format_type ? scan_range_desc.format_type + : rpc_scan_params.format_type; + if (format_type == TFileFormatType::FORMAT_LANCE) { + RETURN_IF_ERROR(read_lance_rows_by_row_ids(scan_range_desc, read_ids, scan_slots, + runtime_state.get(), sub_runtime_profile.get(), + rpc_scan_params, &scan_blocks[idx], + &fetch_statistics[idx])); + } else { + // Parquet/ORC IDs remain signed physical file positions; Lance keeps native uint64 IDs. + std::list legacy_read_ids; + for (const auto row_id : read_ids) { + if (row_id > static_cast(std::numeric_limits::max())) { + return Status::InvalidArgument("legacy external row id exceeds int64 range: {}", + row_id); + } + legacy_read_ids.emplace_back(static_cast(row_id)); + } if (should_use_file_scanner_v2(runtime_state->query_options(), rpc_scan_params, scan_range_desc)) { auto file_scanner = FileScannerV2::create_unique( runtime_state.get(), sub_runtime_profile.get(), &rpc_scan_params, &colname_to_slot_id, &tuple_desc); - RETURN_IF_ERROR(file_scanner->read_by_rows(scan_range_desc, read_ids, &scan_blocks[idx], - &fetch_statistics[idx].init_reader_ms, - &fetch_statistics[idx].get_block_ms)); + RETURN_IF_ERROR(file_scanner->read_by_rows( + scan_range_desc, legacy_read_ids, &scan_blocks[idx], + &fetch_statistics[idx].init_reader_ms, &fetch_statistics[idx].get_block_ms)); } else { // Phase one can still use V1 for table-format variants unsupported by V2, so keep the // matching phase-two path instead of turning a previously valid query into an error. @@ -518,7 +642,7 @@ Status RowIdStorageReader::read_external_row_from_file_mapping( &rpc_scan_params, &colname_to_slot_id, &tuple_desc); RETURN_IF_ERROR(file_scanner->prepare_for_read_lines(scan_range_desc)); RETURN_IF_ERROR(file_scanner->read_lines_from_range( - scan_range_desc, read_ids, &scan_blocks[idx], external_info, + scan_range_desc, legacy_read_ids, &scan_blocks[idx], external_info, &fetch_statistics[idx].init_reader_ms, &fetch_statistics[idx].get_block_ms)); } } @@ -717,6 +841,19 @@ Status RowIdStorageReader::read_batch_external_row( DCHECK(id_file_map->get_external_scan_params().contains(plan_node_id)); const auto* old_scan_params = &(id_file_map->get_external_scan_params().at(plan_node_id)); + // A Lance dataset row ID is meaningful only to Lance take_rows. Paired with a Parquet or + // ORC mapping it would be read as a physical row position of that file. + if (request_block_desc.row_location_version() == + static_cast(ROW_VERSION::LANCE_DATASET_ROW_ID)) { + const auto format_type = first_scan_range_desc.__isset.format_type + ? first_scan_range_desc.format_type + : old_scan_params->format_type; + if (format_type != TFileFormatType::FORMAT_LANCE) { + return Status::InvalidArgument( + "Lance dataset row IDs cannot be fetched with file format {}", + static_cast(format_type)); + } + } std::unordered_map source_column_to_scan_idx; result_column_to_scan_column.reserve(slots.size()); @@ -759,9 +896,9 @@ Status RowIdStorageReader::read_batch_external_row( } // Hash(TFileRangeDesc) => { all the rows that need to be read and their positions in the result block. } + file mapping - // std::multimap : The reason for using multimap is: may need the same row of data multiple times. - std::map, std::shared_ptr>> + // The multimap retains duplicate row IDs because the same source row can appear more than once + // in the materialized result. + std::map, std::shared_ptr>> scan_rows; // Block corresponding to the order of `scan_rows` map. @@ -773,7 +910,21 @@ Status RowIdStorageReader::read_batch_external_row( // Count the time/bytes it takes to read each TFileRangeDesc. (for profile) std::vector fetch_statistics; - auto hash_file_range = [](const TFileRangeDesc& file_range_desc) { + auto hash_file_range = [&rpc_scan_params](const ExternalFileMappingInfo& external_info) { + const auto& file_range_desc = external_info.scan_range_desc; + const auto format_type = file_range_desc.__isset.format_type ? file_range_desc.format_type + : rpc_scan_params.format_type; + if (format_type == TFileFormatType::FORMAT_LANCE) { + // Parquet and ORC row IDs are offsets within a physical file range, so their fetch + // path must keep each path/start_offset pair separate. Lance row IDs instead belong + // to a fixed dataset snapshot. Although phase one registers one file mapping per + // fragment split, phase two uses dataset-level take_rows and can fetch row IDs from + // all of those fragments together. Group the mappings by scan node, snapshot, and + // dataset URI; the plan node keeps independent scans of the same snapshot isolated. + const auto& lance_params = file_range_desc.table_format_params.lance_params; + return fmt::format("lance:{}:{}:{}", external_info.plan_node_id, lance_params.version, + lance_params.dataset_uri); + } std::string value; value.resize(file_range_desc.path.size() + sizeof(file_range_desc.start_offset)); auto* ptr = value.data(); @@ -794,14 +945,12 @@ Status RowIdStorageReader::read_batch_external_row( } const auto& external_info = file_mapping->get_external_file_info(); - const auto& scan_range_desc = external_info.scan_range_desc; - - auto scan_range_hash = hash_file_range(scan_range_desc); + const auto& scan_range_hash = hash_file_range(external_info); if (scan_rows.contains(scan_range_hash)) { scan_rows.at(scan_range_hash).first.emplace(request_block_desc.row_id(j), j); } else { - std::multimap tmp {{request_block_desc.row_id(j), j}}; - scan_rows.emplace(scan_range_hash, std::make_pair(tmp, file_mapping)); + std::multimap rows {{request_block_desc.row_id(j), j}}; + scan_rows.emplace(scan_range_hash, std::make_pair(std::move(rows), file_mapping)); } } @@ -825,8 +974,8 @@ Status RowIdStorageReader::read_batch_external_row( //semaphore: Limit the number of scan tasks submitted at one time std::counting_semaphore semaphore {max_file_scanners}; - std::vector, - std::shared_ptr>> + std::vector< + std::pair, std::shared_ptr>> scan_info_list; scan_info_list.reserve(scan_rows.size()); for (const auto& [_, scan_info] : scan_rows) { @@ -899,6 +1048,8 @@ Status RowIdStorageReader::read_batch_external_row( fmt::memory_buffer file_read_times_buffer; format_to(file_read_times_buffer, "["); + std::map lance_fetch_times_ns; + std::map lance_fetch_counters; size_t idx = 0; for (const auto& [_, scan_info] : scan_rows) { format_to(file_read_lines_buffer, "{}, ", scan_info.first.size()); @@ -906,6 +1057,12 @@ Status RowIdStorageReader::read_batch_external_row( *get_block_avg_ms += fetch_statistics[idx].get_block_ms; format_to(file_read_bytes_buffer, "{}, ", fetch_statistics[idx].file_read_bytes); format_to(file_read_times_buffer, "{}, ", fetch_statistics[idx].file_read_times); + for (const auto& [time_name, time_value] : fetch_statistics[idx].lance_fetch_times_ns) { + lance_fetch_times_ns[time_name] += time_value; + } + for (const auto& [name, value] : fetch_statistics[idx].lance_fetch_counters) { + lance_fetch_counters[name] += value; + } idx++; } @@ -925,6 +1082,14 @@ Status RowIdStorageReader::read_batch_external_row( fmt::to_string(file_read_bytes_buffer)); runtime_profile->add_info_string(FileScannerV2::FileReadTimeProfile, fmt::to_string(file_read_times_buffer)); + for (const auto& [name, value] : lance_fetch_counters) { + runtime_profile->add_info_string( + name, PrettyPrinter::print(value, LanceFetchCountersProfile.at(name))); + } + for (const auto& [time_name, time_value] : lance_fetch_times_ns) { + runtime_profile->add_info_string(time_name, + PrettyPrinter::print(time_value, TUnit::TIME_NS)); + } } runtime_profile->to_proto(pprofile, 2); diff --git a/be/src/exec/rowid_fetcher.h b/be/src/exec/rowid_fetcher.h index 916b77ff214a62..56a75b9098d27f 100644 --- a/be/src/exec/rowid_fetcher.h +++ b/be/src/exec/rowid_fetcher.h @@ -21,9 +21,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -35,6 +37,7 @@ namespace doris { +class RuntimeProfile; class RuntimeState; class TQueryOptions; class TupleDescriptor; @@ -68,6 +71,11 @@ class RowIdStorageReader { static const std::string InitReaderAvgTimeProfile; static const std::string GetBlockAvgTimeProfile; static const std::string FileReadLinesProfile; + static const std::string LanceDatasetOpenTimeProfile; + static const std::string LanceRowIdTakeReadTimeProfile; + static const std::string LanceArrowToDorisBlockTimeProfile; + static const std::string LanceRowIdFetchTotalTimeProfile; + static const std::map LanceFetchCountersProfile; static const std::string TopNLazyMaterializationSecondPhaseLocalIOCount; static const std::string TopNLazyMaterializationSecondPhaseLocalIOBytes; static const std::string TopNLazyMaterializationSecondPhaseRemoteIOCount; @@ -120,8 +128,16 @@ class RowIdStorageReader { Block& result_block, PRuntimeProfileTree* pprofile, int64_t* init_reader_avg_ms, int64_t* get_block_avg_ms, size_t* scan_range_cnt); + static Status read_lance_rows_by_row_ids(const TFileRangeDesc& scan_range_desc, + const std::vector& row_ids, + const std::vector& slots, + RuntimeState* runtime_state, + RuntimeProfile* runtime_profile, + const TFileScanRangeParams& scan_params, Block* block, + ExternalFetchStatistics* fetch_statistics); + static Status read_external_row_from_file_mapping( - size_t idx, const std::multimap& row_ids, + size_t idx, const std::multimap& row_ids, const std::shared_ptr& file_mapping, const std::vector& scan_slots, const TUniqueId& query_id, const std::shared_ptr& runtime_state, std::vector& scan_blocks, @@ -151,6 +167,8 @@ class RowIdStorageReader { struct ExternalFetchStatistics { int64_t init_reader_ms = 0; int64_t get_block_ms = 0; + std::map lance_fetch_times_ns; + std::map lance_fetch_counters; std::string file_read_bytes; std::string file_read_times; }; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 15a71f2c2d1017..3dac8169042c61 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -65,6 +65,7 @@ #include "format_v2/table/hudi_reader.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "format_v2/table/iceberg_reader.h" +#include "format_v2/table/lance_reader.h" #include "format_v2/table/load_reader.h" #include "format_v2/table/paimon_reader.h" #include "format_v2/table/remote_doris_reader.h" @@ -142,6 +143,10 @@ bool is_supported_arrow_table_format(const TFileRangeDesc& range) { return table_format == "remote_doris" || table_format == "adbc"; } +bool is_supported_lance_table_format(const TFileRangeDesc& range) { + return table_format_name(range) == "lance"; +} + bool is_supported_paimon_jni_payload(const TFileRangeDesc& range) { if (!range.__isset.table_format_params || !range.table_format_params.__isset.paimon_params) { return false; @@ -358,6 +363,8 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile return is_supported_table_format(range); } else if (format_type == TFileFormatType::FORMAT_ARROW) { return is_supported_arrow_table_format(range); + } else if (format_type == TFileFormatType::FORMAT_LANCE) { + return is_supported_lance_table_format(range); } else if (format_type == TFileFormatType::FORMAT_JNI) { return is_supported_jni_table_format(range); } else if (is_wal_format(format_type)) { @@ -786,6 +793,8 @@ Status FileScannerV2::_create_table_reader_for_format( } else { *reader = std::make_unique(); } + } else if (table_format == "lance") { + *reader = std::make_unique(); } else if (table_format == "paimon") { *reader = std::make_unique(); } else if (table_format == "hudi") { @@ -1095,6 +1104,9 @@ Status FileScannerV2::_to_file_format(TFileFormatType::type format_type, case TFileFormatType::FORMAT_WAL: *file_format = format::FileFormat::WAL; return Status::OK(); + case TFileFormatType::FORMAT_LANCE: + *file_format = format::FileFormat::LANCE; + return Status::OK(); default: return Status::NotSupported("FileScannerV2 does not support file format {}", to_string(format_type)); diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 98258447a03249..e00a476df0bdc0 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -60,6 +60,7 @@ enum class FileFormat { JNI, ARROW, WAL, + LANCE, }; struct FileScanRequest { diff --git a/be/src/format_v2/lance/lance_reader_helper.cpp b/be/src/format_v2/lance/lance_reader_helper.cpp new file mode 100644 index 00000000000000..81876f0097bcf1 --- /dev/null +++ b/be/src/format_v2/lance/lance_reader_helper.cpp @@ -0,0 +1,802 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_reader_helper.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nothing.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type_serde/arrow_validation.h" +#include "exec/common/endian.h" + +namespace doris::format::lance { + +RuntimeProfile::Counter* add_lance_counter(RuntimeProfile* profile, const char* name, + TUnit::type unit, const char* group) { + // These counters label sections; they do not represent time or a sum of their children. + ADD_CHILD_COUNTER_WITH_LEVEL(profile, group, TUnit::NONE, LANCE_READER_PROFILE, 1); + return ADD_CHILD_COUNTER_WITH_LEVEL(profile, name, unit, group, 1); +} + +namespace { + +constexpr std::string_view ARROW_EXTENSION_NAME = "ARROW:extension:name"; +constexpr std::string_view ARROW_JSON_EXTENSION = "arrow.json"; +constexpr std::string_view LANCE_JSON_EXTENSION = "lance.json"; +constexpr std::string_view LANCE_BFLOAT16_EXTENSION = "lance.bfloat16"; + +enum class LanceExtensionKind { + NONE, + JSON, + BFLOAT16, +}; + +int arrow_time_precision(arrow::TimeUnit::type unit) { + switch (unit) { + case arrow::TimeUnit::SECOND: + return 0; + case arrow::TimeUnit::MILLI: + return 3; + case arrow::TimeUnit::MICRO: + case arrow::TimeUnit::NANO: + return 6; + } + return 6; +} + +// Extract and validate extension names from metadata and registered Arrow types. +Status get_lance_extension(const std::shared_ptr& field, + LanceExtensionKind* extension_kind, + std::shared_ptr* storage_type) { + DORIS_CHECK(field != nullptr); + DORIS_CHECK(extension_kind != nullptr); + DORIS_CHECK(storage_type != nullptr); + *extension_kind = LanceExtensionKind::NONE; + *storage_type = field->type(); + + std::optional extension_name; + if (field->HasMetadata()) { + const auto metadata_name = field->metadata()->Get(ARROW_EXTENSION_NAME); + if (metadata_name.ok() && !metadata_name.ValueUnsafe().empty()) { + extension_name = metadata_name.ValueUnsafe(); + } + } + if (field->type()->id() == arrow::Type::EXTENSION) { + const auto extension_type = std::dynamic_pointer_cast(field->type()); + if (extension_type == nullptr) { + return Status::InvalidArgument("invalid Arrow extension type for Lance field '{}'", + field->name()); + } + if (extension_name.has_value() && *extension_name != extension_type->extension_name()) { + return Status::InvalidArgument( + "conflicting Arrow extension names for Lance field '{}': '{}' and '{}'", + field->name(), *extension_name, extension_type->extension_name()); + } + extension_name = extension_type->extension_name(); + *storage_type = extension_type->storage_type(); + } + if (!extension_name.has_value()) { + return Status::OK(); + } + + if (*extension_name == ARROW_JSON_EXTENSION) { + if ((*storage_type)->id() != arrow::Type::STRING && + (*storage_type)->id() != arrow::Type::LARGE_STRING) { + return Status::NotSupported( + "Arrow JSON extension for Lance field '{}' requires UTF8 storage, got {}", + field->name(), (*storage_type)->ToString()); + } + *extension_kind = LanceExtensionKind::JSON; + return Status::OK(); + } + if (*extension_name == LANCE_JSON_EXTENSION) { + if ((*storage_type)->id() != arrow::Type::LARGE_BINARY) { + return Status::NotSupported( + "Lance JSON extension for field '{}' requires LARGE_BINARY storage, got {}", + field->name(), (*storage_type)->ToString()); + } + *extension_kind = LanceExtensionKind::JSON; + return Status::OK(); + } + if (*extension_name == LANCE_BFLOAT16_EXTENSION) { + if ((*storage_type)->id() != arrow::Type::FIXED_SIZE_BINARY || + std::static_pointer_cast(*storage_type)->byte_width() != + 2) { + return Status::NotSupported( + "Lance BFloat16 extension for field '{}' requires FIXED_SIZE_BINARY(2) " + "storage, got {}", + field->name(), (*storage_type)->ToString()); + } + *extension_kind = LanceExtensionKind::BFLOAT16; + return Status::OK(); + } + return Status::NotSupported("unsupported Lance Arrow extension type '{}' for field '{}'", + *extension_name, field->name()); +} + +// Map Arrow fields recursively, preserving Null leaves in complex types. +Status arrow_field_to_doris_type(const std::shared_ptr& field, + DataTypePtr* doris_type) { + const auto nullable_primitive = [&](PrimitiveType type, int precision = 0, int scale = 0, + int len = -1) { + *doris_type = + DataTypeFactory::instance().create_data_type(type, true, precision, scale, len); + return Status::OK(); + }; + + LanceExtensionKind extension_kind; + std::shared_ptr arrow_type; + RETURN_IF_ERROR(get_lance_extension(field, &extension_kind, &arrow_type)); + switch (extension_kind) { + case LanceExtensionKind::JSON: + return nullable_primitive(TYPE_JSONB); + case LanceExtensionKind::BFLOAT16: + return nullable_primitive(TYPE_FLOAT); + case LanceExtensionKind::NONE: + break; + } + if (arrow_type->id() == arrow::Type::DICTIONARY) { + return Status::NotSupported("unsupported Lance Arrow dictionary type for field '{}': {}", + field->name(), arrow_type->ToString()); + } + + switch (arrow_type->id()) { + case arrow::Type::NA: + // Required Null leaves can bypass NullableSerDe through FE schema reconstruction. + // Reject them before Arrow NA buffers reach the Boolean-backed physical SerDe. + if (!field->nullable()) { + return Status::NotSupported("non-nullable Lance Arrow Null field: {}", field->name()); + } + return nullable_primitive(TYPE_NULL); + case arrow::Type::BOOL: + return nullable_primitive(TYPE_BOOLEAN); + case arrow::Type::INT8: + return nullable_primitive(TYPE_TINYINT); + case arrow::Type::UINT8: + case arrow::Type::INT16: + return nullable_primitive(TYPE_SMALLINT); + case arrow::Type::UINT16: + case arrow::Type::INT32: + return nullable_primitive(TYPE_INT); + case arrow::Type::UINT32: + case arrow::Type::INT64: + return nullable_primitive(TYPE_BIGINT); + case arrow::Type::UINT64: + return nullable_primitive(TYPE_LARGEINT); + case arrow::Type::HALF_FLOAT: + case arrow::Type::FLOAT: + return nullable_primitive(TYPE_FLOAT); + case arrow::Type::DOUBLE: + return nullable_primitive(TYPE_DOUBLE); + case arrow::Type::STRING: + case arrow::Type::LARGE_STRING: + return nullable_primitive(TYPE_STRING); + case arrow::Type::BINARY: + case arrow::Type::LARGE_BINARY: + return nullable_primitive(TYPE_VARBINARY, 0, 0, std::numeric_limits::max()); + case arrow::Type::FIXED_SIZE_BINARY: { + const auto binary = std::static_pointer_cast(arrow_type); + return nullable_primitive(TYPE_VARBINARY, 0, 0, binary->byte_width()); + } + case arrow::Type::DATE32: + case arrow::Type::DATE64: + return nullable_primitive(TYPE_DATEV2); + case arrow::Type::TIME32: + case arrow::Type::TIME64: { + const auto time = std::static_pointer_cast(arrow_type); + return nullable_primitive(TYPE_TIMEV2, 0, arrow_time_precision(time->unit())); + } + case arrow::Type::TIMESTAMP: { + const auto timestamp = std::static_pointer_cast(arrow_type); + const auto timestamp_type = + timestamp->timezone().empty() ? TYPE_DATETIMEV2 : TYPE_TIMESTAMPTZ; + return nullable_primitive(timestamp_type, 0, arrow_time_precision(timestamp->unit())); + } + case arrow::Type::DURATION: + return nullable_primitive(TYPE_BIGINT); + case arrow::Type::DECIMAL128: + case arrow::Type::DECIMAL256: { + const auto decimal = std::static_pointer_cast(arrow_type); + const int precision = decimal->precision(); + const int scale = decimal->scale(); + if (precision <= 0 || precision > arrow::Decimal256Type::kMaxPrecision || scale < 0 || + scale > precision) { + return Status::NotSupported( + "unsupported Lance Arrow decimal type for field '{}': precision={}, scale={}", + field->name(), precision, scale); + } + const PrimitiveType doris_decimal_type = precision <= 9 ? TYPE_DECIMAL32 + : precision <= 18 ? TYPE_DECIMAL64 + : precision <= 38 ? TYPE_DECIMAL128I + : TYPE_DECIMAL256; + return nullable_primitive(doris_decimal_type, precision, scale); + } + case arrow::Type::LIST: + case arrow::Type::LARGE_LIST: + case arrow::Type::FIXED_SIZE_LIST: { + const auto list = std::static_pointer_cast(arrow_type); + DataTypePtr value_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(list->value_field(), &value_type)); + *doris_type = make_nullable(std::make_shared(value_type)); + return Status::OK(); + } + case arrow::Type::MAP: { + const auto map = std::static_pointer_cast(arrow_type); + DataTypePtr key_type; + DataTypePtr item_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(map->key_field(), &key_type)); + RETURN_IF_ERROR(arrow_field_to_doris_type(map->item_field(), &item_type)); + *doris_type = make_nullable(std::make_shared(key_type, item_type)); + return Status::OK(); + } + case arrow::Type::STRUCT: { + const auto struct_type = std::static_pointer_cast(arrow_type); + DataTypes field_types; + Strings field_names; + field_types.reserve(struct_type->num_fields()); + field_names.reserve(struct_type->num_fields()); + for (const auto& child : struct_type->fields()) { + DataTypePtr field_type; + RETURN_IF_ERROR(arrow_field_to_doris_type(child, &field_type)); + field_types.emplace_back(std::move(field_type)); + field_names.emplace_back(child->name()); + } + *doris_type = make_nullable(std::make_shared(field_types, field_names)); + return Status::OK(); + } + default: + return Status::NotSupported("unsupported Lance Arrow type: {}", arrow_type->ToString()); + } +} + +// Widen little-endian Lance BFloat16 values to Arrow Float32 without precision loss. +Status convert_bfloat16_array(const std::shared_ptr& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr* normalized) { + DORIS_CHECK(array != nullptr); + DORIS_CHECK(memory_pool != nullptr); + DORIS_CHECK(normalized != nullptr); + const auto fixed_binary = std::dynamic_pointer_cast(array); + if (fixed_binary == nullptr || fixed_binary->byte_width() != 2) { + return Status::InvalidArgument("invalid Lance BFloat16 array storage: {}", + array->type()->ToString()); + } + if (config::enable_arrow_input_validation) { + check_arrow_fixed_width_buffer(*fixed_binary, sizeof(uint16_t)); + } + + arrow::FloatBuilder builder(memory_pool); + auto arrow_status = builder.Reserve(fixed_binary->length()); + if (!arrow_status.ok()) { + return Status::InternalError("reserve Lance BFloat16 output failed: {}", + arrow_status.message()); + } + for (int64_t row = 0; row < fixed_binary->length(); ++row) { + if (fixed_binary->IsNull(row)) { + arrow_status = builder.AppendNull(); + } else { + const auto bits = LittleEndian::Load16(fixed_binary->GetValue(row)); + arrow_status = builder.Append(std::bit_cast(static_cast(bits) << 16)); + } + if (!arrow_status.ok()) { + return Status::InternalError("append Lance BFloat16 value failed: {}", + arrow_status.message()); + } + } + std::shared_ptr result; + arrow_status = builder.Finish(&result); + if (!arrow_status.ok()) { + return Status::InternalError("finish Lance BFloat16 conversion failed: {}", + arrow_status.message()); + } + *normalized = std::move(result); + return Status::OK(); +} + +// Narrow a Decimal256 array whose declared precision fits Decimal128. The safe cast keeps every +// in-precision value and its scale, and rejects a value that exceeds the declared precision +// instead of truncating it: unscaled 10^18 stored in a decimal256(18, 2) field fails with +// "Decimal value does not fit in precision 18". +Status narrow_decimal256_array(std::string_view field_name, + const std::shared_ptr& array, + const std::shared_ptr& decimal128_type, + arrow::MemoryPool* memory_pool, + std::shared_ptr* normalized) { + DORIS_CHECK(array != nullptr); + DORIS_CHECK(memory_pool != nullptr); + DORIS_CHECK(normalized != nullptr); + arrow::compute::ExecContext exec_context(memory_pool); + auto narrowed = arrow::compute::Cast(*array, decimal128_type, + arrow::compute::CastOptions::Safe(), &exec_context); + if (!narrowed.ok()) { + return Status::InvalidArgument("narrow Lance Decimal256 field '{}' to {} failed: {}", + field_name, decimal128_type->ToString(), + narrowed.status().message()); + } + *normalized = std::move(narrowed).ValueUnsafe(); + return Status::OK(); +} + +// Rebuild a nested Arrow type after one or more child arrays changed physical type. +Status set_lance_nested_type(std::string_view field_name, + const std::shared_ptr& source_type, + const arrow::FieldVector& child_fields, + std::shared_ptr* data) { + switch (source_type->id()) { + case arrow::Type::LIST: + (*data)->type = arrow::list(child_fields[0]); + break; + case arrow::Type::LARGE_LIST: + (*data)->type = arrow::large_list(child_fields[0]); + break; + case arrow::Type::FIXED_SIZE_LIST: + (*data)->type = arrow::fixed_size_list( + child_fields[0], + std::static_pointer_cast(source_type)->list_size()); + break; + case arrow::Type::STRUCT: + (*data)->type = arrow::struct_(child_fields); + break; + case arrow::Type::MAP: { + const auto map_type = std::static_pointer_cast(source_type); + auto normalized_type = arrow::MapType::Make(child_fields[0], map_type->keys_sorted()); + if (!normalized_type.ok()) { + return Status::InvalidArgument("normalize Lance map field '{}' failed: {}", field_name, + normalized_type.status().message()); + } + (*data)->type = std::move(normalized_type).ValueUnsafe(); + break; + } + default: + return Status::InvalidArgument("Lance field '{}' has unexpected child-bearing type {}", + field_name, source_type->ToString()); + } + return Status::OK(); +} + +// Materialize the visible range into an offset-zero Arrow array for Doris SerDes. +Status compact_lance_array(const std::shared_ptr& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr* compacted) { + DORIS_CHECK(memory_pool != nullptr); + const auto validation = array->ValidateFull(); + if (!validation.ok()) { + return Status::InvalidArgument("validate sliced Lance array failed: {}", + validation.message()); + } + auto builder_result = arrow::MakeBuilder(array->type(), memory_pool); + if (!builder_result.ok()) { + return Status::InternalError("create sliced Lance array builder failed: {}", + builder_result.status().message()); + } + auto builder = std::move(builder_result).ValueUnsafe(); + auto arrow_status = builder->Reserve(array->length()); + if (!arrow_status.ok()) { + return Status::InternalError("reserve sliced Lance array builder failed: {}", + arrow_status.message()); + } + arrow_status = builder->AppendArraySlice(*array->data(), 0, array->length()); + if (!arrow_status.ok()) { + return Status::InternalError("copy sliced Lance array failed: {}", arrow_status.message()); + } + arrow_status = builder->Finish(compacted); + if (!arrow_status.ok()) { + return Status::InternalError("finish sliced Lance array copy failed: {}", + arrow_status.message()); + } + if ((*compacted)->offset() != 0) { + return Status::InternalError("compacted Lance array retained offset {}", + (*compacted)->offset()); + } + return Status::OK(); +} + +// Compact a sliced variable-offset parent and all of its visible descendants. +template +Status compact_lance_offset_array(const std::shared_ptr& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr* compacted) { + const auto offset_array = std::dynamic_pointer_cast(array); + if (offset_array == nullptr) { + return Status::InvalidArgument("invalid sliced Lance offset array: {}", + array->type()->ToString()); + } + const auto child_begin = static_cast(offset_array->value_offset(0)); + const auto child_end = static_cast(offset_array->value_offset(offset_array->length())); + const auto& values = offset_array->values(); + if (child_begin < 0 || child_end < child_begin || child_end > values->length()) { + return Status::InvalidArgument("invalid sliced Lance offsets [{}, {}) for child length {}", + child_begin, child_end, values->length()); + } + if (array->offset() == 0 && child_begin == 0 && child_end == values->length()) { + *compacted = array; + return Status::OK(); + } + return compact_lance_array(array, memory_pool, compacted); +} + +// Compact sliced arrays and nested children only when Doris cannot consume their current layout. +Status compact_lance_array_if_needed(const std::shared_ptr& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr* compacted) { + switch (array->type_id()) { + case arrow::Type::LIST: + return compact_lance_offset_array(array, memory_pool, compacted); + case arrow::Type::LARGE_LIST: + return compact_lance_offset_array(array, memory_pool, compacted); + case arrow::Type::MAP: + return compact_lance_offset_array(array, memory_pool, compacted); + case arrow::Type::FIXED_SIZE_LIST: { + const auto list = std::dynamic_pointer_cast(array); + if (list == nullptr) { + return Status::InvalidArgument("invalid sliced Lance fixed-size list array: {}", + array->type()->ToString()); + } + const auto child_begin = list->value_offset(0); + const auto child_length = list->length() * list->value_length(); + const auto& values = list->values(); + if (child_begin < 0 || child_length < 0 || child_begin > values->length() - child_length) { + return Status::InvalidArgument( + "invalid sliced Lance fixed-size list range [{}, {}) for child length {}", + child_begin, child_begin + child_length, values->length()); + } + if (array->offset() == 0 && child_begin == 0 && child_length == values->length()) { + *compacted = array; + return Status::OK(); + } + return compact_lance_array(array, memory_pool, compacted); + } + case arrow::Type::STRUCT: { + const auto struct_array = std::dynamic_pointer_cast(array); + if (struct_array == nullptr) { + return Status::InvalidArgument("invalid sliced Lance struct array: {}", + array->type()->ToString()); + } + bool requires_compaction = array->offset() != 0; + for (const auto& child : array->data()->child_data) { + requires_compaction |= child->length != array->length(); + } + if (!requires_compaction) { + *compacted = array; + return Status::OK(); + } + return compact_lance_array(array, memory_pool, compacted); + } + default: + if (array->offset() == 0) { + *compacted = array; + return Status::OK(); + } + return compact_lance_array(array, memory_pool, compacted); + } +} + +} // namespace + +Status LanceArrowArrayNormalizer::create(const std::shared_ptr& field, + LanceArrowArrayNormalizer* normalizer) { + DORIS_CHECK(field != nullptr); + DORIS_CHECK(normalizer != nullptr); + + LanceExtensionKind extension_kind; + std::shared_ptr storage_type; + RETURN_IF_ERROR(get_lance_extension(field, &extension_kind, &storage_type)); + + LanceArrowArrayNormalizer result; + result._field_name = field->name(); + result._storage_type = std::move(storage_type); + result._unwrap_registered_extension = field->type()->id() == arrow::Type::EXTENSION; + result._convert_bfloat16 = extension_kind == LanceExtensionKind::BFLOAT16; + // Lance keeps the 256-bit carrier of pa.decimal256(18, 2), while the FE and + // arrow_field_to_doris_type() type the column DECIMAL64 by precision. The DECIMAL32/64/128I + // SerDes read only Decimal128Array, so a Decimal256Array reaching them made their dynamic_cast + // return null and the BE crash on the first value, also for nested leaves. Precision 39..76 + // stays Decimal256 and maps to DECIMAL256. + if (result._storage_type->id() == arrow::Type::DECIMAL256) { + const auto& decimal = static_cast(*result._storage_type); + if (decimal.precision() <= arrow::Decimal128Type::kMaxPrecision) { + result._decimal128_type = arrow::decimal128(decimal.precision(), decimal.scale()); + } + } + result._requires_special_handling = result._unwrap_registered_extension || + result._convert_bfloat16 || + result._decimal128_type != nullptr; + result._child_normalizers.reserve(result._storage_type->num_fields()); + for (const auto& child_field : result._storage_type->fields()) { + LanceArrowArrayNormalizer child_normalizer; + RETURN_IF_ERROR(create(child_field, &child_normalizer)); + result._requires_special_handling |= child_normalizer._requires_special_handling; + result._child_normalizers.emplace_back(std::move(child_normalizer)); + } + *normalizer = std::move(result); + return Status::OK(); +} + +Status LanceArrowArrayNormalizer::normalize_for_doris( + const std::shared_ptr& array, arrow::MemoryPool* memory_pool, + std::shared_ptr* normalized) const { + DORIS_CHECK(array != nullptr); + DORIS_CHECK(memory_pool != nullptr); + DORIS_CHECK(normalized != nullptr); + + // The schema binding already resolved extension metadata and nested special types. For common + // arrays, runtime work is limited to checking whether the visible slice needs compaction. + if (!_requires_special_handling) { + return compact_lance_array_if_needed(array, memory_pool, normalized); + } + + auto storage_array = array; + if (_unwrap_registered_extension && array->type_id() == arrow::Type::EXTENSION) { + const auto extension_array = std::dynamic_pointer_cast(array); + if (extension_array == nullptr) { + return Status::InvalidArgument("invalid Arrow extension array: {}", + array->type()->ToString()); + } + storage_array = extension_array->storage(); + } + // Extension and BFloat16 arrays are first converted to their physical storage representation. + if (storage_array->type_id() != _storage_type->id()) { + return Status::InvalidArgument( + "Lance field '{}' storage type {} does not match array type {}", _field_name, + _storage_type->ToString(), storage_array->type()->ToString()); + } + if (_convert_bfloat16) { + return convert_bfloat16_array(storage_array, memory_pool, normalized); + } + if (_decimal128_type != nullptr) { + // The cast reads only the visible slice and returns an offset-zero array, so no + // compaction follows. For a nested leaf, the parent's call swaps in this array in the + // child loop below and rebuilds its own type through set_lance_nested_type(). + return narrow_decimal256_array(_field_name, storage_array, _decimal128_type, memory_pool, + normalized); + } + + // Normalize extension children before compacting the parent. Arrow cannot create a builder + // for list/struct types whose child is still an ExtensionType (Arrow 24 returns + // NotImplemented), while the rebuilt physical type can be compacted normally. + const auto& child_data = storage_array->data()->child_data; + if (_child_normalizers.size() != child_data.size()) { + return Status::InvalidArgument( + "Lance field '{}' has {} child fields but its Arrow array has {} children", + _field_name, _child_normalizers.size(), child_data.size()); + } + + arrow::FieldVector normalized_fields; + std::shared_ptr normalized_data; + for (size_t child_idx = 0; child_idx < _child_normalizers.size(); ++child_idx) { + const auto& child_normalizer = _child_normalizers[child_idx]; + if (!child_normalizer._requires_special_handling) { + continue; + } + auto child_array = arrow::MakeArray(child_data[child_idx]); + std::shared_ptr normalized_child; + RETURN_IF_ERROR( + child_normalizer.normalize_for_doris(child_array, memory_pool, &normalized_child)); + if (normalized_child.get() == child_array.get()) { + continue; + } + if (normalized_data == nullptr) { + normalized_data = storage_array->data()->Copy(); + normalized_fields = storage_array->type()->fields(); + } + normalized_data->child_data[child_idx] = normalized_child->data(); + normalized_fields[child_idx] = + normalized_fields[child_idx]->WithType(normalized_child->type()); + } + if (normalized_data != nullptr) { + RETURN_IF_ERROR(set_lance_nested_type(_field_name, storage_array->type(), normalized_fields, + &normalized_data)); + storage_array = arrow::MakeArray(std::move(normalized_data)); + } + + // Compact only after child types are physical, so MakeBuilder never sees a nested + // ExtensionType. This also preserves the existing zero-copy path when no slice is present. + return compact_lance_array_if_needed(storage_array, memory_pool, normalized); +} + +#ifdef BE_TEST +// Expose Lance Arrow normalization for allocation-sensitive unit tests. +Status normalize_lance_arrow_array_for_test(const std::shared_ptr& field, + const std::shared_ptr& array, + std::shared_ptr* normalized, + arrow::MemoryPool* memory_pool) { + LanceArrowArrayNormalizer normalizer; + RETURN_IF_ERROR(LanceArrowArrayNormalizer::create(field, &normalizer)); + return normalizer.normalize_for_doris( + array, memory_pool != nullptr ? memory_pool : arrow::default_memory_pool(), normalized); +} +#endif + +Status import_lance_dataset_schema(LanceDataset* dataset, std::shared_ptr* schema) { + DORIS_CHECK(dataset != nullptr); + DORIS_CHECK(schema != nullptr); + + ArrowSchema arrow_schema {}; + if (lance_dataset_schema(dataset, &arrow_schema) != 0) { + return lance_error("get Lance dataset schema"); + } + auto imported_schema = arrow::ImportSchema(&arrow_schema); + if (!imported_schema.ok()) { + if (arrow_schema.release != nullptr) { + arrow_schema.release(&arrow_schema); + } + return Status::InternalError("import Lance Arrow schema failed: {}", + imported_schema.status().message()); + } + *schema = std::move(imported_schema).ValueUnsafe(); + return Status::OK(); +} + +void LanceDatasetDeleter::operator()(LanceDataset* dataset) const { + lance_dataset_close(dataset); +} + +void LanceScannerDeleter::operator()(LanceScanner* scanner) const { + lance_scanner_close(scanner); +} + +void LanceBatchDeleter::operator()(LanceBatch* batch) const { + lance_batch_free(batch); +} + +size_t lance_vector_element_width(TVectorElementType::type type) { + switch (type) { + case TVectorElementType::FLOAT16: + return sizeof(uint16_t); + case TVectorElementType::FLOAT32: + return sizeof(float); + case TVectorElementType::FLOAT64: + return sizeof(double); + case TVectorElementType::UINT8: + case TVectorElementType::INT8: + return sizeof(uint8_t); + } + return 0; +} + +Status parse_fragment_ids(const TLanceFileDesc& lance_params, std::vector* fragment_ids) { + DORIS_CHECK(fragment_ids != nullptr); + fragment_ids->clear(); + if (!lance_params.__isset.fragment_ids || lance_params.fragment_ids.empty()) { + return Status::OK(); + } + fragment_ids->reserve(lance_params.fragment_ids.size()); + for (const auto fragment_id : lance_params.fragment_ids) { + if (fragment_id < 0) { + return Status::InvalidArgument("Lance fragment id must be non-negative: {}", + fragment_id); + } + fragment_ids->emplace_back(static_cast(fragment_id)); + } + return Status::OK(); +} + +Status parse_index_segment_uuids(const TLanceFileDesc& lance_params, + std::vector* segment_uuids, size_t* segment_count) { + DORIS_CHECK(segment_uuids != nullptr); + DORIS_CHECK(segment_count != nullptr); + segment_uuids->clear(); + *segment_count = 0; + if (!lance_params.__isset.index_segment_uuids || lance_params.index_segment_uuids.empty()) { + return Status::OK(); + } + constexpr size_t UUID_SIZE = 16; + if (lance_params.index_segment_uuids.size() > std::numeric_limits::max() / UUID_SIZE) { + return Status::InvalidArgument("too many Lance index segment UUIDs"); + } + segment_uuids->reserve(lance_params.index_segment_uuids.size() * UUID_SIZE); + for (const auto& uuid : lance_params.index_segment_uuids) { + if (uuid.size() != UUID_SIZE) { + return Status::InvalidArgument("Lance index segment UUID must contain 16 bytes, got {}", + uuid.size()); + } + segment_uuids->insert(segment_uuids->end(), uuid.begin(), uuid.end()); + } + *segment_count = lance_params.index_segment_uuids.size(); + return Status::OK(); +} + +Status convert_arrow_schema_to_doris(const std::shared_ptr& arrow_schema, + std::vector* column_names, + std::vector* column_types) { + DORIS_CHECK(arrow_schema != nullptr); + DORIS_CHECK(column_names != nullptr); + DORIS_CHECK(column_types != nullptr); + + std::vector parsed_names; + std::vector parsed_types; + parsed_names.reserve(arrow_schema->num_fields()); + parsed_types.reserve(arrow_schema->num_fields()); + std::unordered_set unique_names; + unique_names.reserve(arrow_schema->num_fields()); + for (const auto& field : arrow_schema->fields()) { + if (!unique_names.emplace(field->name()).second) { + return Status::InvalidArgument("duplicate Lance schema column: {}", field->name()); + } + DataTypePtr doris_type; + const auto type_status = arrow_field_to_doris_type(field, &doris_type); + if (type_status.is()) { + parsed_types.emplace_back(std::make_shared()); + } else { + RETURN_IF_ERROR(type_status); + DORIS_CHECK(doris_type != nullptr); + parsed_types.emplace_back(std::move(doris_type)); + } + parsed_names.emplace_back(field->name()); + } + *column_names = std::move(parsed_names); + *column_types = std::move(parsed_types); + return Status::OK(); +} + +Status build_lance_storage_options(const TFileScanRangeParams* scan_params, + std::vector* options) { + DORIS_CHECK(options != nullptr); + options->clear(); + if (scan_params == nullptr || !scan_params->__isset.lance_scan_params || + !scan_params->lance_scan_params.__isset.lance_storage_options) { + return Status::OK(); + } + const auto& storage_options = scan_params->lance_scan_params.lance_storage_options; + options->reserve(storage_options.size() * 2); + for (const auto& [key, value] : storage_options) { + // Both values cross a C-string boundary. Reject embedded NULs instead of silently opening + // a different dataset configuration from the one validated and used by the FE. + if (key.find('\0') != std::string::npos || value.find('\0') != std::string::npos) { + return Status::InvalidArgument( + "Lance storage option '{}' contains a NUL and cannot reach lance-c", + key.substr(0, key.find('\0'))); + } + options->emplace_back(key); + options->emplace_back(value); + } + return Status::OK(); +} + +Status lance_error(std::string_view operation) { + const char* raw_message = lance_last_error_message(); + std::string message = raw_message == nullptr ? "" : raw_message; + if (raw_message != nullptr) { + lance_free_string(raw_message); + } + if (message.empty()) { + message = fmt::format("error_code={}", static_cast(lance_last_error_code())); + } + return Status::InternalError("{} failed: {}", operation, message); +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_reader_helper.h b/be/src/format_v2/lance/lance_reader_helper.h new file mode 100644 index 00000000000000..decc3d5dc63b8b --- /dev/null +++ b/be/src/format_v2/lance/lance_reader_helper.h @@ -0,0 +1,320 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "gen_cpp/PlanNodes_types.h" +#include "runtime/runtime_profile.h" + +struct LanceBatch; +struct LanceDataset; +struct LanceScanner; + +namespace arrow { +class Array; +class DataType; +class Field; +class MemoryPool; +class Schema; +} // namespace arrow + +namespace doris::format::lance { + +inline constexpr std::string_view LANCE_DISTANCE_COLUMN = "_distance"; +inline constexpr std::string_view LANCE_SCORE_COLUMN = "_score"; +inline constexpr std::string_view LANCE_ROW_ID_COLUMN = "_rowid"; +inline constexpr const char* LANCE_READER_PROFILE = "LanceReader"; +inline constexpr const char* LANCE_TIMING_PROFILE = "LanceTiming"; +inline constexpr const char* LANCE_IO_PROFILE = "LanceIO"; +inline constexpr const char* LANCE_INDEX_PROFILE = "LanceIndex"; +inline constexpr const char* LANCE_VECTOR_INDEX_PROFILE = "LanceVectorIndex"; +inline constexpr const char* LANCE_SCALAR_INDEX_PROFILE = "LanceScalarIndex"; +inline constexpr const char* LANCE_EXECUTION_PLAN_PROFILE = "LanceExecutionPlan"; +inline constexpr const char* LANCE_SCAN_PROFILE = "LanceScan"; +inline constexpr const char* LANCE_SEARCH_PLAN_PROFILE = "LanceSearchPlan"; +inline constexpr const char* LANCE_DATA_CACHE_PROFILE = "LanceDataCache"; + +struct LanceProfileMetric { + std::string_view native_name; + const char* profile_name; + const char* group; + TUnit::type unit; +}; + +// Lance invokes the statistics callback from its execution streams. Keep the metric definitions +// immutable and pre-register their counters before installing the callback. The callback then +// updates only metrics reported by Lance without changing the profile counter tree. +inline constexpr LanceProfileMetric LANCE_SCAN_METRICS[] = { + {.native_name = "index_cache_hits", + .profile_name = "LanceIndexCacheHits", + .group = LANCE_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "index_cache_misses", + .profile_name = "LanceIndexCacheMisses", + .group = LANCE_INDEX_PROFILE, + .unit = TUnit::UNIT}, + // Row-ID prefilter loader metrics are separate from ANN/FTS result counts. + {.native_name = "prefilter_loads", + .profile_name = "RowIdPrefilterLoads", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "prefilter_input_rows", + .profile_name = "RowIdPrefilterInputRows", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "prefilter_input_batches", + .profile_name = "RowIdPrefilterInputBatches", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "prefilter_row_ids", + .profile_name = "RowIdPrefilterIds", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::UNIT}, + // LoadTime includes input polling and set construction, so it overlaps component timers. + {.native_name = "prefilter_load_time", + .profile_name = "RowIdPrefilterLoadTime", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "prefilter_input_time", + .profile_name = "RowIdPrefilterInputTime", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "prefilter_build_time", + .profile_name = "RowIdPrefilterBuildTime", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "fragments_scanned", + .profile_name = "LanceFragmentsScanned", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "ranges_scanned", + .profile_name = "LanceRowOffsetRangesScanned", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "rows_scanned", + .profile_name = "LanceRowsScanned", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::UNIT}, + // This is Lance scan execution wait time, not Doris scanner scheduling wait time. + {.native_name = "task_wait_time", + .profile_name = "LanceScanTaskWaitTime", + .group = LANCE_SCAN_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "partitions_ranked", + .profile_name = "LanceIVFPartitionsConsidered", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + // Lance also reports partitions_searched for FTS, so keep it in the generic index group. + {.native_name = "partitions_searched", + .profile_name = "LanceIndexPartitionsSearched", + .group = LANCE_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "deltas_searched", + .profile_name = "LanceVectorIndexDeltasSearched", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + // These partition stages accumulate across concurrent work and can overlap each other. + {.native_name = "find_partitions_elapsed", + .profile_name = "LanceIVFPartitionSelectionTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "scalar_segments_requested", + .profile_name = "LanceScalarIndexSegmentsRequested", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "scalar_segments_searched", + .profile_name = "LanceScalarIndexSegmentsSearched", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "scalar_segment_fallbacks", + .profile_name = "LanceScalarIndexSegmentFallbacks", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "scalar_segment_candidate_rows", + .profile_name = "LanceScalarIndexCandidateRows", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "scalar_segment_prepare_time", + .profile_name = "LanceScalarIndexCandidateEvaluationTime", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "scalar_segment_search_time", + .profile_name = "LanceScalarIndexSegmentSearchTime", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "search_time", + .profile_name = "LanceScalarIndexQueryTime", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "serialization_time", + .profile_name = "LanceScalarIndexResultSerializationTime", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + // Index partition times overlap; DistanceTopK includes candidate filtering, scoring, + // and heap updates. + {.native_name = "index_open_time", + .profile_name = "LanceIndexOpenTime", + .group = LANCE_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "index_partition_load_time", + .profile_name = "LanceIndexPartitionLoadTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "index_partition_prepare_time", + .profile_name = "LanceIndexPartitionPrepareTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "index_prefilter_wait_time", + .profile_name = "LanceIndexPrefilterWaitTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "index_cpu_queue_wait_time", + .profile_name = "LanceIndexCpuQueueWaitTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "index_search_time", + .profile_name = "LanceIndexSearchTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "index_query_prepare_time", + .profile_name = "LanceIndexQueryPrepareTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "index_distance_topk_time", + .profile_name = "LanceIndexDistanceTopKTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "index_result_materialize_time", + .profile_name = "LanceIndexResultMaterializeTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + // These execution-plan stage timers are also cumulative across concurrent tasks. + {.native_name = "ANNIVFPartitionExec_elapsed_compute", + .profile_name = "LanceANNPartitionExecTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "ANNSubIndexExec_elapsed_compute", + .profile_name = "LanceANNSubIndexExecTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "ANNIvfBatchExec_elapsed_compute", + .profile_name = "LanceANNBatchExecTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "SortExec_elapsed_compute", + .profile_name = "LanceSortComputeTime", + .group = LANCE_EXECUTION_PLAN_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "SortPreservingMergeExec_elapsed_compute", + .profile_name = "LanceSortMergeComputeTime", + .group = LANCE_EXECUTION_PLAN_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "TakeExec_elapsed_compute", + .profile_name = "LanceTakeExecTime", + .group = LANCE_EXECUTION_PLAN_PROFILE, + .unit = TUnit::TIME_NS}, + {.native_name = "KNNVectorDistanceExec_elapsed_compute", + .profile_name = "LanceVectorDistanceComputeTime", + .group = LANCE_VECTOR_INDEX_PROFILE, + .unit = TUnit::TIME_NS}, +}; + +RuntimeProfile::Counter* add_lance_counter(RuntimeProfile* profile, const char* name, + TUnit::type unit, const char* group); + +struct LanceDatasetDeleter { + void operator()(LanceDataset* dataset) const; +}; + +struct LanceScannerDeleter { + void operator()(LanceScanner* scanner) const; +}; + +struct LanceBatchDeleter { + void operator()(LanceBatch* batch) const; +}; + +size_t lance_vector_element_width(TVectorElementType::type type); + +// Import the physical Arrow schema owned by a Lance dataset. The caller owns the resulting +// shared schema and may cache it for operations that need physical Arrow types. +Status import_lance_dataset_schema(LanceDataset* dataset, std::shared_ptr* schema); + +// Validate and convert the fragment and index-segment identifiers carried by the FE into the +// unsigned and packed representations expected by lance-c. +Status parse_fragment_ids(const TLanceFileDesc& lance_params, std::vector* fragment_ids); +Status parse_index_segment_uuids(const TLanceFileDesc& lance_params, + std::vector* segment_uuids, size_t* segment_count); + +// Resolves one Arrow field once when a stream schema is bound. Runtime conversion then reuses this +// plan instead of rescanning extension metadata and nested fields for every array. +class LanceArrowArrayNormalizer { +public: + static Status create(const std::shared_ptr& field, + LanceArrowArrayNormalizer* normalizer); + + // Return an Arrow array whose physical layout and type can be consumed by Doris SerDes. The + // input is returned unchanged when no compaction or type adaptation is required. + Status normalize_for_doris(const std::shared_ptr& array, + arrow::MemoryPool* memory_pool, + std::shared_ptr* normalized) const; + +private: + std::string _field_name; + std::shared_ptr _storage_type; + // decimal128(p, s) for a Decimal256 field with p <= 38, which Doris types DECIMAL32/64/128I + // by precision; null for every other field. + std::shared_ptr _decimal128_type; + std::vector _child_normalizers; + bool _unwrap_registered_extension = false; + bool _convert_bfloat16 = false; + bool _requires_special_handling = false; +}; + +#ifdef BE_TEST +// Expose Lance Arrow normalization for allocation-sensitive unit tests. +Status normalize_lance_arrow_array_for_test(const std::shared_ptr& field, + const std::shared_ptr& array, + std::shared_ptr* normalized, + arrow::MemoryPool* memory_pool = nullptr); +#endif + +// Convert every top-level field without discarding unsupported columns. Malformed schemas still +// return an error and leave both output vectors unchanged. DataTypeNothing is the local sentinel +// for a valid Arrow field whose logical type Doris does not support. +Status convert_arrow_schema_to_doris(const std::shared_ptr& arrow_schema, + std::vector* column_names, + std::vector* column_types); + +// The FE sends storage options in Lance's own vocabulary. Preserve the key-value sequence exactly +// while validating that every value can cross the C-string boundary into lance-c. +Status build_lance_storage_options(const TFileScanRangeParams* scan_params, + std::vector* options); + +// Copy and release lance-c's thread-local error message before returning a Doris status. +Status lance_error(std::string_view operation); + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_record_batch_converter.cpp b/be/src/format_v2/lance/lance_record_batch_converter.cpp new file mode 100644 index 00000000000000..7f0bf848a08dcd --- /dev/null +++ b/be/src/format_v2/lance/lance_record_batch_converter.cpp @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_record_batch_converter.h" + +#include +#include +#include +#include + +#include + +#include "common/consts.h" +#include "common/exception.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_nullable.h" +#include "format_v2/lance/lance_reader_helper.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "storage/utils.h" + +namespace doris::format::lance { +namespace { + +constexpr size_t IGNORED_OUTPUT_INDEX = std::numeric_limits::max(); + +} // namespace + +arrow::MemoryPool* LanceRecordBatchConverter::_query_arrow_memory_pool( + RuntimeState* runtime_state) { + DORIS_CHECK(runtime_state != nullptr); + if (runtime_state->exec_env() != nullptr) { + if (auto* memory_pool = runtime_state->exec_env()->arrow_memory_pool(); + memory_pool != nullptr) { + return memory_pool; + } + } + return arrow::default_memory_pool(); +} + +Status LanceRecordBatchConverter::init(RuntimeState* runtime_state, + const std::vector& projected_columns, + SearchKind search_kind) { + DORIS_CHECK(runtime_state != nullptr); + _search_kind = search_kind; + _timezone = runtime_state->timezone_obj(); + _memory_pool = _query_arrow_memory_pool(runtime_state); + _output_columns.clear(); + _output_columns.reserve(projected_columns.size()); + _output_index_by_input_name.clear(); + _output_index_by_input_name.reserve(projected_columns.size()); + _global_rowid_output_index.reset(); + reset_schema(); + + for (const auto& column : projected_columns) { + if (column.type == nullptr) { + return Status::InvalidArgument("Lance projected column '{}' has no type", column.name); + } + if (column.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) { + if (_search_kind == SearchKind::NORMAL) { + return Status::NotSupported( + "Lance global row id is currently supported only for external search"); + } + if (_global_rowid_output_index.has_value()) { + return Status::InvalidArgument("duplicate Lance global row id projected column: {}", + column.name); + } + if (remove_nullable(column.type)->get_primitive_type() != TYPE_STRING) { + return Status::InvalidArgument( + "Lance global row id column '{}' must have Doris STRING type, but was {}", + column.name, column.type->get_name()); + } + _global_rowid_output_index = _output_columns.size(); + // Lance returns the native `_rowid`, while Doris exposes it through a generated + // global-rowid column. One input column cannot also satisfy a regular `_rowid` + // projection, so reject an existing binding instead of silently replacing it. + if (!_output_index_by_input_name + .emplace(LANCE_ROW_ID_COLUMN, *_global_rowid_output_index) + .second) { + return Status::InvalidArgument( + "Lance global row id cannot be combined with projected column '{}'", + LANCE_ROW_ID_COLUMN); + } + _output_columns.push_back({column.name, nullptr}); + continue; + } + const auto output_index = _output_columns.size(); + if (!_output_index_by_input_name.emplace(column.name, output_index).second) { + return Status::InvalidArgument("duplicate Lance projected column: {}", column.name); + } + if (_search_kind == SearchKind::VECTOR && column.name == LANCE_DISTANCE_COLUMN && + remove_nullable(column.type)->get_primitive_type() != TYPE_FLOAT) { + return Status::InvalidArgument( + "Lance vector search column '{}' must have Doris FLOAT type, but was {}", + LANCE_DISTANCE_COLUMN, column.type->get_name()); + } + if (_search_kind == SearchKind::FULL_TEXT && column.name == LANCE_SCORE_COLUMN && + remove_nullable(column.type)->get_primitive_type() != TYPE_FLOAT) { + return Status::InvalidArgument( + "Lance full-text search column '{}' must have Doris FLOAT type, but was {}", + LANCE_SCORE_COLUMN, column.type->get_name()); + } + _output_columns.push_back({column.name, column.type->get_serde()}); + } + return Status::OK(); +} + +Status LanceRecordBatchConverter::_bind_schema(const std::shared_ptr& schema) { + DORIS_CHECK(schema != nullptr); + if (_bound_schema == schema || + (_bound_schema != nullptr && _bound_schema->Equals(*schema, true))) { + return Status::OK(); + } + + std::vector arrow_column_to_output(static_cast(schema->num_fields()), + IGNORED_OUTPUT_INDEX); + std::vector array_normalizer_for_arrow_column( + static_cast(schema->num_fields())); + std::vector output_has_source_column(_output_columns.size(), false); + for (int arrow_index = 0; arrow_index < schema->num_fields(); ++arrow_index) { + const auto& field = schema->field(arrow_index); + const auto output = _output_index_by_input_name.find(field->name()); + if (output == _output_index_by_input_name.end()) { + const bool is_omitted_search_result = + (_search_kind == SearchKind::VECTOR && + field->name() == LANCE_DISTANCE_COLUMN) || + (_search_kind == SearchKind::FULL_TEXT && field->name() == LANCE_SCORE_COLUMN); + // Lance may return this generated result even when Doris did not project it. + if (is_omitted_search_result) { + continue; + } + return Status::InternalError("Lance returned unknown column '{}'", field->name()); + } + const auto output_index = output->second; + if (output_has_source_column[output_index]) { + return Status::InternalError("Lance returned duplicate column '{}'", field->name()); + } + output_has_source_column[output_index] = true; + arrow_column_to_output[arrow_index] = output_index; + RETURN_IF_ERROR(LanceArrowArrayNormalizer::create( + field, &array_normalizer_for_arrow_column[static_cast(arrow_index)])); + } + for (size_t output_index = 0; output_index < _output_columns.size(); ++output_index) { + if (!output_has_source_column[output_index]) { + return Status::InternalError("Lance did not return requested column '{}'", + _output_columns[output_index].name); + } + } + _bound_schema = schema; + _arrow_column_to_output_index = std::move(arrow_column_to_output); + _array_normalizer_for_arrow_column = std::move(array_normalizer_for_arrow_column); + return Status::OK(); +} + +Status LanceRecordBatchConverter::convert_record_batch_to_block( + const std::shared_ptr& record_batch, Block* block, + const std::optional& global_rowid_context, size_t* rows) { + DORIS_CHECK(record_batch != nullptr); + DORIS_CHECK(block != nullptr); + DORIS_CHECK(rows != nullptr); + RETURN_IF_ERROR(_bind_schema(record_batch->schema())); + + const auto row_count = static_cast(record_batch->num_rows()); + auto columns_guard = block->mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + for (int arrow_index = 0; arrow_index < record_batch->num_columns(); ++arrow_index) { + const auto output_index = _arrow_column_to_output_index[static_cast(arrow_index)]; + if (output_index == IGNORED_OUTPUT_INDEX) { + continue; + } + const auto& output = _output_columns[output_index]; + const auto& arrow_column = record_batch->column(arrow_index); + if (is_global_rowid_output(output_index)) { + if (!global_rowid_context.has_value()) { + return Status::InvalidArgument( + "Lance global row id requested without global row id context"); + } + RETURN_IF_ERROR(_append_global_row_ids(arrow_column, columns[output_index], + *global_rowid_context)); + continue; + } + const auto& field = record_batch->schema()->field(arrow_index); + try { + if (arrow_column->type_id() == arrow::Type::NA) { + columns[output_index]->insert_many_defaults(row_count); + continue; + } + std::shared_ptr normalized_column; + RETURN_IF_ERROR( + _array_normalizer_for_arrow_column[static_cast(arrow_index)] + .normalize_for_doris(arrow_column, _memory_pool, &normalized_column)); + RETURN_IF_ERROR(output.serde->read_column_from_arrow( + *columns[output_index], normalized_column.get(), 0, row_count, _timezone)); + } catch (const Exception& e) { + return Status::InternalError("convert Lance Arrow column '{}' failed: {}", + field->name(), e.what()); + } + } + *rows = row_count; + return Status::OK(); +} + +void LanceRecordBatchConverter::reset_schema() { + _bound_schema.reset(); + _arrow_column_to_output_index.clear(); + _array_normalizer_for_arrow_column.clear(); +} + +Status LanceRecordBatchConverter::_append_global_row_ids( + const std::shared_ptr& row_ids, MutableColumnPtr& output_column, + const GlobalRowIdContext& context) const { + if (row_ids->type_id() != arrow::Type::UINT64) { + return Status::InternalError("Lance row id column must be Arrow UINT64, but was {}", + row_ids->type()->ToString()); + } + + ColumnString* data_column = nullptr; + ColumnUInt8::Container* null_map = nullptr; + if (auto* nullable = check_and_get_column(*output_column)) { + data_column = check_and_get_column(nullable->get_nested_column()); + null_map = &nullable->get_null_map_data(); + } else { + data_column = check_and_get_column(*output_column); + } + if (data_column == nullptr) { + return Status::InternalError("Lance global row id output column must be STRING"); + } + + const auto typed_row_ids = std::static_pointer_cast(row_ids); + if (typed_row_ids->null_count() != 0) { + return Status::InternalError("Lance returned null row id"); + } + const auto row_count = static_cast(typed_row_ids->length()); + if (null_map != nullptr) { + null_map->resize_fill(null_map->size() + row_count, 0); + } + for (size_t row = 0; row < row_count; ++row) { + const GlobalRowLoacationV2 location(ROW_VERSION::LANCE_DATASET_ROW_ID, context.backend_id, + context.file_id, typed_row_ids->Value(row)); + data_column->insert_data(reinterpret_cast(&location), sizeof(location)); + } + return Status::OK(); +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_record_batch_converter.h b/be/src/format_v2/lance/lance_record_batch_converter.h new file mode 100644 index 00000000000000..491d480a26fe8f --- /dev/null +++ b/be/src/format_v2/lance/lance_record_batch_converter.h @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type_serde/data_type_serde.h" +#include "format_v2/column_data.h" +#include "format_v2/lance/lance_reader_helper.h" + +namespace arrow { +class Array; +class MemoryPool; +class RecordBatch; +class Schema; +} // namespace arrow + +namespace doris { +class RuntimeState; +} + +namespace doris::format::lance { + +enum class SearchKind { NORMAL, VECTOR, FULL_TEXT }; + +// Bridges one Lance Arrow record-batch stream into an existing Doris Block. It owns the stable +// output-column metadata and lazily binds an incoming Arrow schema once, so ordinary batches only +// perform positional conversion. Each Arrow stream has an independent schema binding. +class LanceRecordBatchConverter { +public: + Status init(RuntimeState* runtime_state, const std::vector& projected_columns, + SearchKind search_kind); + + Status convert_record_batch_to_block( + const std::shared_ptr& record_batch, Block* block, + const std::optional& global_rowid_context, size_t* rows); + + bool requires_global_rowid() const { return _global_rowid_output_index.has_value(); } + bool is_global_rowid_output(size_t output_index) const { + return _global_rowid_output_index.has_value() && + *_global_rowid_output_index == output_index; + } + void reset_schema(); + +private: + struct OutputColumn { + std::string name; + DataTypeSerDeSPtr serde; + }; + + static arrow::MemoryPool* _query_arrow_memory_pool(RuntimeState* runtime_state); + Status _bind_schema(const std::shared_ptr& schema); + Status _append_global_row_ids(const std::shared_ptr& row_ids, + MutableColumnPtr& output_column, + const GlobalRowIdContext& context) const; + + SearchKind _search_kind = SearchKind::NORMAL; + cctz::time_zone _timezone; + arrow::MemoryPool* _memory_pool = nullptr; + std::vector _output_columns; + std::unordered_map _output_index_by_input_name; + std::optional _global_rowid_output_index; + std::shared_ptr _bound_schema; + std::vector _arrow_column_to_output_index; + std::vector _array_normalizer_for_arrow_column; +}; + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_runtime_filter_helper.cpp b/be/src/format_v2/lance/lance_runtime_filter_helper.cpp new file mode 100644 index 00000000000000..a04c8ce278a4c9 --- /dev/null +++ b/be/src/format_v2/lance/lance_runtime_filter_helper.cpp @@ -0,0 +1,550 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_runtime_filter_helper.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" +#include "core/data_type/data_type_nullable.h" +#include "core/field.h" +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format/format_common.h" +#include "runtime/runtime_profile.h" +#include "util/utf8_check.h" + +namespace doris::format::lance { +namespace { + +constexpr std::string_view LANCE_RUNTIME_FILTER_CACHE_KEY_PREFIX = "lance-runtime-filter-sql:"; + +std::string format_filter_ids(const std::vector& filter_ids) { + std::string result; + for (const auto filter_id : filter_ids) { + if (!result.empty()) { + result.append(","); + } + result.append(std::to_string(filter_id)); + } + return result; +} + +std::string quote_sql_identifier(std::string_view identifier) { + // Lance SQL uses backticks for delimited identifiers. Escape an embedded backtick by doubling + // it, matching the SQL parser's quoted-identifier syntax. + std::string quoted("`"); + quoted.reserve(identifier.size() + 2); + for (const char ch : identifier) { + if (ch == '`') { + quoted.append("``"); + } else { + quoted.push_back(ch); + } + } + quoted.push_back('`'); + return quoted; +} + +const RuntimeFilterExpr* get_runtime_filter(const VExprContextSPtr& conjunct) { + if (conjunct == nullptr || conjunct->root() == nullptr) { + return nullptr; + } + return dynamic_cast(conjunct->root().get()); +} + +std::string lowercase_ascii(std::string value) { + std::ranges::transform(value, value.begin(), [](const unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + return value; +} + +std::string quote_string_value(std::string_view value) { + std::string quoted("'"); + quoted.reserve(value.size() + 2); + for (const char ch : value) { + if (ch == '\'') { + quoted.append("''"); + } else { + quoted.push_back(ch); + } + } + quoted.push_back('\''); + return quoted; +} + +std::shared_ptr find_unique_physical_type(const arrow::Schema& schema, + std::string_view column_name) { + const auto fields = schema.GetAllFieldsByName(column_name); + return fields.size() == 1 ? fields.front()->type() : nullptr; +} + +std::optional parse_int64(std::string_view value) { + int64_t parsed = 0; + const auto* begin = value.data(); + const auto* end = begin + value.size(); + const auto result = std::from_chars(begin, end, parsed); + if (result.ec != std::errc() || result.ptr != end) { + return std::nullopt; + } + return parsed; +} + +bool integer_literal_fits_decimal_precision(std::string_view value, int32_t precision) { + if (!value.empty() && (value.front() == '-' || value.front() == '+')) { + value.remove_prefix(1); + } + const auto first_nonzero = value.find_first_not_of('0'); + if (first_nonzero == std::string_view::npos) { + return true; + } + return value.size() - first_nonzero <= static_cast(precision); +} + +// This checks Lance planner coercion, not whether stored Lance values fit their own Arrow type. +// Unsigned Arrow columns map to wider signed Doris types (for example UInt8 -> SMALLINT), so an RF +// produced from another input may contain a valid Doris value that cannot be coerced back to the +// narrower physical type. +bool integer_literal_fits_physical_type(std::string_view value, + const arrow::DataType& physical_type) { + const auto parsed = parse_int64(value); + if (!parsed.has_value()) { + return false; + } + const auto literal = *parsed; + switch (physical_type.id()) { + case arrow::Type::INT8: + return literal >= std::numeric_limits::min() && + literal <= std::numeric_limits::max(); + case arrow::Type::UINT8: + return literal >= 0 && literal <= std::numeric_limits::max(); + case arrow::Type::INT16: + return literal >= std::numeric_limits::min() && + literal <= std::numeric_limits::max(); + case arrow::Type::UINT16: + return literal >= 0 && literal <= std::numeric_limits::max(); + case arrow::Type::INT32: + return literal >= std::numeric_limits::min() && + literal <= std::numeric_limits::max(); + case arrow::Type::UINT32: + return literal >= 0 && literal <= std::numeric_limits::max(); + case arrow::Type::INT64: + return true; + case arrow::Type::UINT64: + // The pinned Lance SQL parser first parses a bare integer as i64. Values above INT64_MAX + // become Float64, which cannot be safely coerced to UInt64. + return literal >= 0; + default: + return false; + } +} + +bool physical_type_supports_literal(PrimitiveType logical_type, std::string_view value, + const arrow::DataType& physical_type) { + switch (logical_type) { + case TYPE_BOOLEAN: + return physical_type.id() == arrow::Type::BOOL; + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_LARGEINT: + return integer_literal_fits_physical_type(value, physical_type); + case TYPE_FLOAT: + // Arrow HALF_FLOAT and FLOAT both materialize as Doris FLOAT, but the pinned Lance planner + // cannot coerce an SQL Int64/Float64 literal to Float16. + return physical_type.id() == arrow::Type::FLOAT; + case TYPE_DOUBLE: + return physical_type.id() == arrow::Type::DOUBLE; + case TYPE_DECIMALV2: + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMAL128I: + case TYPE_DECIMAL256: { + if (physical_type.id() != arrow::Type::DECIMAL128 && + physical_type.id() != arrow::Type::DECIMAL256) { + return false; + } + const auto& decimal = static_cast(physical_type); + // A scale-bearing bare SQL token is parsed as Float64, which the pinned planner cannot + // coerce to Decimal. Scale-zero values are safe only while they parse exactly as i64. + return decimal.scale() == 0 && parse_int64(value).has_value() && + integer_literal_fits_decimal_precision(value, decimal.precision()); + } + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + return (physical_type.id() == arrow::Type::STRING || + physical_type.id() == arrow::Type::LARGE_STRING) && + value.find('\0') == std::string_view::npos && + validate_utf8(value.data(), value.size()); + case TYPE_DATE: + case TYPE_DATEV2: + // Date64 is only specified to contain a millisecond count that should be day-aligned. + // Doris materialization discards any sub-day remainder, so only Date32 is unconditionally + // comparison-equivalent. + return physical_type.id() == arrow::Type::DATE32; + case TYPE_DATETIME: + case TYPE_DATETIMEV2: { + if (physical_type.id() != arrow::Type::TIMESTAMP) { + return false; + } + const auto& timestamp = static_cast(physical_type); + // Doris DATETIMEV2 stores at most microseconds, so a nanosecond predicate is not equivalent + // to evaluating the residual after Arrow-to-Doris materialization. + return timestamp.timezone().empty() && timestamp.unit() != arrow::TimeUnit::NANO; + } + default: + return false; + } +} + +std::optional to_lance_sql_literal(const VLiteral& literal, + const arrow::DataType& physical_type) { + const auto type = remove_nullable(literal.get_data_type()); + auto options = DataTypeSerDe::get_default_format_options(); + auto timezone = cctz::utc_time_zone(); + options.timezone = &timezone; + const auto value = literal.value(options); + if (!physical_type_supports_literal(type->get_primitive_type(), value, physical_type)) { + // The SQL literal would either fail Lance planning or compare different physical values + // from the Doris residual. Keep the RF residual-only in either case. + return std::nullopt; + } + switch (type->get_primitive_type()) { + case TYPE_BOOLEAN: { + const auto normalized = lowercase_ascii(value); + if (normalized == "0" || normalized == "false") { + return "FALSE"; + } + if (normalized == "1" || normalized == "true") { + return "TRUE"; + } + // Do not pass an implementation-specific boolean spelling through to Lance SQL. + return std::nullopt; + } + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_LARGEINT: + return value; + case TYPE_FLOAT: + case TYPE_DOUBLE: { + const auto normalized = lowercase_ascii(value); + if (normalized.find("nan") != std::string::npos || + normalized.find("inf") != std::string::npos) { + // Lance SQL has no portable scalar spelling with Doris-equivalent NaN/Inf semantics. + return std::nullopt; + } + return value; + } + case TYPE_DECIMALV2: + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMAL128I: + case TYPE_DECIMAL256: + return value; + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: + return quote_string_value(value); + case TYPE_DATE: + case TYPE_DATEV2: + return "DATE " + quote_string_value(value); + case TYPE_DATETIME: + case TYPE_DATETIMEV2: + return "TIMESTAMP " + quote_string_value(value); + default: + return std::nullopt; + } +} + +template +std::optional in_value_to_lance_sql_literal(const void* raw_value, + const DataTypePtr& data_type, + const arrow::DataType& physical_type) { + if (raw_value == nullptr || data_type == nullptr) { + // A null set entry needs SQL three-valued semantics and cannot be serialized as a scalar. + return std::nullopt; + } + Field field; + if constexpr (is_string_type(PT)) { + const auto* value = static_cast(raw_value); + using CppType = typename PrimitiveTypeTraits::CppType; + field = Field::create_field(CppType(value->data, value->size)); + } else { + using CppType = typename PrimitiveTypeTraits::CppType; + field = Field::create_field(*static_cast(raw_value)); + } + return to_lance_sql_literal(VLiteral(data_type, field), physical_type); +} + +std::optional in_value_to_lance_sql_literal(PrimitiveType primitive_type, + const void* raw_value, + const DataTypePtr& data_type, + const arrow::DataType& physical_type) { +#define DISPATCH_IN_VALUE(TYPE) \ + case TYPE: \ + return in_value_to_lance_sql_literal(raw_value, data_type, physical_type) + switch (primitive_type) { + DISPATCH_IN_VALUE(TYPE_BOOLEAN); + DISPATCH_IN_VALUE(TYPE_TINYINT); + DISPATCH_IN_VALUE(TYPE_SMALLINT); + DISPATCH_IN_VALUE(TYPE_INT); + DISPATCH_IN_VALUE(TYPE_BIGINT); + DISPATCH_IN_VALUE(TYPE_LARGEINT); + DISPATCH_IN_VALUE(TYPE_FLOAT); + DISPATCH_IN_VALUE(TYPE_DOUBLE); + DISPATCH_IN_VALUE(TYPE_DATE); + DISPATCH_IN_VALUE(TYPE_DATETIME); + DISPATCH_IN_VALUE(TYPE_DATEV2); + DISPATCH_IN_VALUE(TYPE_DATETIMEV2); + DISPATCH_IN_VALUE(TYPE_CHAR); + DISPATCH_IN_VALUE(TYPE_VARCHAR); + DISPATCH_IN_VALUE(TYPE_STRING); + DISPATCH_IN_VALUE(TYPE_DECIMALV2); + DISPATCH_IN_VALUE(TYPE_DECIMAL32); + DISPATCH_IN_VALUE(TYPE_DECIMAL64); + DISPATCH_IN_VALUE(TYPE_DECIMAL128I); + DISPATCH_IN_VALUE(TYPE_DECIMAL256); + default: + return std::nullopt; + } +#undef DISPATCH_IN_VALUE +} + +std::optional build_in_filter_sql(const VDirectInPredicate& predicate, + const arrow::Schema& physical_schema) { + if (predicate.get_num_children() != 1) { + return std::nullopt; + } + const auto slot = std::dynamic_pointer_cast(predicate.get_child(0)); + const auto values = predicate.get_set_func(); + if (slot == nullptr || slot->data_type() == nullptr || values == nullptr) { + // Only a direct slot-backed IN set has a stable Lance column/literal representation. + return std::nullopt; + } + if (values->contain_null()) { + // Dropping NULL would change the Doris IN-filter semantics. + return std::nullopt; + } + + const auto data_type = remove_nullable(slot->data_type()); + const auto physical_type = find_unique_physical_type(physical_schema, slot->column_name()); + if (physical_type == nullptr) { + // A missing or ambiguous top-level field cannot be bound safely by column name. + return std::nullopt; + } + std::string expression("(" + quote_sql_identifier(slot->column_name()) + " IN ("); + auto* iterator = values->begin(); + bool first_value = true; + while (iterator != nullptr && iterator->has_next()) { + auto value = in_value_to_lance_sql_literal( + data_type->get_primitive_type(), iterator->get_value(), data_type, *physical_type); + if (!value.has_value()) { + // Partially pushing an IN set would introduce false negatives, so reject it as a unit. + return std::nullopt; + } + if (!first_value) { + expression.append(", "); + } + expression.append(*value); + first_value = false; + iterator->next(); + } + if (first_value) { + // An empty IN set has no direct Lance SQL representation. + return std::nullopt; + } + expression.append("))"); + return expression; +} + +std::optional build_range_filter_sql(const VExpr& predicate, + const arrow::Schema& physical_schema) { + if ((predicate.op() != TExprOpcode::GE && predicate.op() != TExprOpcode::LE) || + predicate.get_num_children() != 2) { + return std::nullopt; + } + const auto slot = std::dynamic_pointer_cast(predicate.get_child(0)); + const auto literal = std::dynamic_pointer_cast(predicate.get_child(1)); + if (slot == nullptr || literal == nullptr) { + // Computed operands are intentionally left to Doris instead of reconstructing expression + // SQL. + return std::nullopt; + } + const auto slot_type = remove_nullable(slot->data_type())->get_primitive_type(); + if (predicate.op() == TExprOpcode::GE && + (slot_type == TYPE_FLOAT || slot_type == TYPE_DOUBLE)) { + // Doris orders every NaN above all other values regardless of its sign + // (GreaterThanEqualsFloat in common/compare.h), so `x >= 1` keeps NaN rows. Lance + // evaluates the pushed SQL with Arrow/DataFusion kernels that use IEEE 754 totalOrder, + // where a negative NaN such as 0xffc00000 sorts below every finite value: `x >= 1` drops + // that row before the Doris residual can keep it. A MIN-only filter, for example from a + // nested-loop join on `x >= y`, would lose such rows. `<=` only lets negative NaNs + // through to the residual, which removes them, and IN sets with NaN members are not + // pushed. PaimonRustPredicateConverter keeps floating predicates residual-only too. + return std::nullopt; + } + const auto physical_type = find_unique_physical_type(physical_schema, slot->column_name()); + if (physical_type == nullptr) { + // A missing or ambiguous top-level field cannot be bound safely by column name. + return std::nullopt; + } + const auto sql_literal = to_lance_sql_literal(*literal, *physical_type); + if (!sql_literal.has_value()) { + // Preserve the Doris residual when its bound has no proven-equivalent physical literal. + return std::nullopt; + } + const auto* sql_operator = predicate.op() == TExprOpcode::GE ? ">=" : "<="; + return "(" + quote_sql_identifier(slot->column_name()) + " " + sql_operator + " " + + *sql_literal + ")"; +} + +std::optional runtime_filter_to_lance_sql(const RuntimeFilterExpr& runtime_filter, + const arrow::Schema& physical_schema) { + if (runtime_filter.is_null_aware()) { + // Doris restores probe NULLs to true for null-aware wrappers; ordinary Lance comparisons + // would discard those rows before the residual can restore them. + return std::nullopt; + } + const auto impl = runtime_filter.get_impl(); + if (impl == nullptr) { + // A published wrapper without an implementation has nothing safe to translate. + return std::nullopt; + } + if (const auto* in_predicate = dynamic_cast(impl.get()); + in_predicate != nullptr) { + return build_in_filter_sql(*in_predicate, physical_schema); + } + return build_range_filter_sql(*impl, physical_schema); +} + +std::shared_ptr build_runtime_filter_sql( + const VExprContextSPtrs& conjuncts, const arrow::Schema& physical_schema) { + auto result = std::make_shared(); + std::set seen_filter_ids; + std::set pushed_filter_ids; + for (const auto& conjunct : conjuncts) { + const auto* runtime_filter = get_runtime_filter(conjunct); + if (runtime_filter == nullptr) { + continue; + } + const auto filter_id = runtime_filter->filter_id(); + seen_filter_ids.emplace(filter_id); + + const auto expression = runtime_filter_to_lance_sql(*runtime_filter, physical_schema); + if (!expression.has_value()) { + continue; + } + if (!result->expression.empty()) { + result->expression.append(" AND "); + } + result->expression.append(*expression); + pushed_filter_ids.emplace(filter_id); + } + + result->pushable_filter_ids.assign(pushed_filter_ids.begin(), pushed_filter_ids.end()); + for (const auto filter_id : seen_filter_ids) { + if (!pushed_filter_ids.contains(filter_id)) { + result->skipped_filter_ids.emplace_back(filter_id); + } + } + return result; +} + +std::optional build_cache_key(const VExprContextSPtrs& conjuncts) { + // This cache is scoped to one FileScanLocalState. An RF is immutable after it is published, so + // its sorted IDs identify the snapshot shared by parallel readers. + std::set filter_ids; + for (const auto& conjunct : conjuncts) { + if (const auto* runtime_filter = get_runtime_filter(conjunct); runtime_filter != nullptr) { + filter_ids.emplace(runtime_filter->filter_id()); + } + } + if (filter_ids.empty()) { + // Avoid allocating a cache entry for conjunct snapshots that contain no runtime filters. + return std::nullopt; + } + + std::string key(LANCE_RUNTIME_FILTER_CACHE_KEY_PREFIX); + for (const auto filter_id : filter_ids) { + key.append(std::to_string(filter_id)).append(","); + } + return key; +} + +} // namespace + +bool has_lance_runtime_filters(const VExprContextSPtrs& conjuncts) { + return std::ranges::any_of(conjuncts, [](const auto& conjunct) { + return get_runtime_filter(conjunct) != nullptr; + }); +} + +std::shared_ptr get_or_create_lance_runtime_filter_sql( + const VExprContextSPtrs& conjuncts, const arrow::Schema& physical_schema, + ShardedKVCache* cache) { + const auto cache_key = build_cache_key(conjuncts); + if (!cache_key.has_value()) { + return nullptr; + } + if (cache == nullptr) { + return build_runtime_filter_sql(conjuncts, physical_schema); + } + + auto* cached = cache->get>( + *cache_key, [&]() -> std::shared_ptr* { + return new std::shared_ptr( + build_runtime_filter_sql(conjuncts, physical_schema)); + }); + return cached == nullptr ? nullptr : *cached; +} + +void record_lance_runtime_filter_pushdown(RuntimeProfile* profile, + const LanceRuntimeFilterSql& runtime_filter_sql) { + DORIS_CHECK(profile != nullptr); + const auto pushed_ids = format_filter_ids(runtime_filter_sql.pushable_filter_ids); + const auto skipped_ids = format_filter_ids(runtime_filter_sql.skipped_filter_ids); + + if (!pushed_ids.empty()) { + profile->add_info_string("LanceRuntimeFilterPushedIds", pushed_ids); + } + if (!skipped_ids.empty()) { + profile->add_info_string("LanceRuntimeFilterSkippedIds", skipped_ids); + } + VLOG_DEBUG << "Lance runtime filter pushdown: pushed_ids=[" << pushed_ids << "], skipped_ids=[" + << skipped_ids << "]"; +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_runtime_filter_helper.h b/be/src/format_v2/lance/lance_runtime_filter_helper.h new file mode 100644 index 00000000000000..40529951ccc7c8 --- /dev/null +++ b/be/src/format_v2/lance/lance_runtime_filter_helper.h @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "exprs/vexpr_fwd.h" + +namespace arrow { +class Schema; +} + +namespace doris { + +class ShardedKVCache; +class RuntimeProfile; + +namespace format::lance { + +struct LanceRuntimeFilterSql { + std::string expression; + std::vector pushable_filter_ids; + std::vector skipped_filter_ids; +}; + +// Return whether the conjunct list contains at least one Doris runtime filter. This lets callers +// avoid importing Lance's physical Arrow schema when no runtime-filter SQL can be produced. +bool has_lance_runtime_filters(const VExprContextSPtrs& conjuncts); + +// Build one immutable SQL snapshot for all supported Doris runtime filters. When cache is non-null, +// equivalent RF snapshots from parallel Lance readers in the same FileScanLocalState share the +// conversion result. The returned snapshot also identifies RFs that cannot be represented exactly +// by Lance SQL. A null result means that conjuncts contain no Doris runtime filter. +std::shared_ptr get_or_create_lance_runtime_filter_sql( + const VExprContextSPtrs& conjuncts, const arrow::Schema& physical_schema, + ShardedKVCache* cache); + +void record_lance_runtime_filter_pushdown(RuntimeProfile* profile, + const LanceRuntimeFilterSql& runtime_filter_sql); + +} // namespace format::lance +} // namespace doris diff --git a/be/src/format_v2/lance/lance_session_manager.cpp b/be/src/format_v2/lance/lance_session_manager.cpp new file mode 100644 index 00000000000000..884eb5db18c5e7 --- /dev/null +++ b/be/src/format_v2/lance/lance_session_manager.cpp @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_session_manager.h" + +#include + +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "common/metrics/doris_metrics.h" +#include "common/metrics/metrics.h" +#include "format_v2/lance/lance_reader_helper.h" + +namespace doris::format::lance { +namespace { + +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_capacity_bytes, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_usage_bytes, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_entries, MetricUnit::NOUNIT); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_hits_total, MetricUnit::OPERATIONS); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(lance_session_index_cache_misses_total, + MetricUnit::OPERATIONS); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_capacity_bytes, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_usage_bytes, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_entries, MetricUnit::NOUNIT); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_hits_total, + MetricUnit::OPERATIONS); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(lance_session_metadata_cache_misses_total, + MetricUnit::OPERATIONS); + +constexpr std::string_view LANCE_SESSION_CACHE_METRICS_HOOK = "lance_session_cache"; + +int64_t metric_value(uint64_t value) { + return static_cast( + std::min(value, static_cast(std::numeric_limits::max()))); +} + +LanceSessionManager::Config load_lance_session_config() { + return { + .lance_index_cache_size_bytes = config::lance_index_cache_size_bytes, + .lance_metadata_cache_size_bytes = config::lance_metadata_cache_size_bytes, + .enable_lance_data_cache = config::enable_lance_data_cache, + .lance_data_cache_path = config::lance_data_cache_path, + .lance_data_cache_disk_capacity_bytes = config::lance_data_cache_disk_capacity_bytes, + .lance_data_cache_read_block_size_bytes = + config::lance_data_cache_read_block_size_bytes, + }; +} + +} // namespace + +class LanceSessionMetrics final { +public: + LanceSessionMetrics(LanceSession* session, const LanceSessionManager::Config& config) + : _session(session), _entity(DorisMetrics::instance()->server_entity()) { + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_index_cache_capacity_bytes); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_index_cache_usage_bytes); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_index_cache_entries); + INT_COUNTER_METRIC_REGISTER(_entity, lance_session_index_cache_hits_total); + INT_COUNTER_METRIC_REGISTER(_entity, lance_session_index_cache_misses_total); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_metadata_cache_capacity_bytes); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_metadata_cache_usage_bytes); + INT_GAUGE_METRIC_REGISTER(_entity, lance_session_metadata_cache_entries); + INT_COUNTER_METRIC_REGISTER(_entity, lance_session_metadata_cache_hits_total); + INT_COUNTER_METRIC_REGISTER(_entity, lance_session_metadata_cache_misses_total); + + lance_session_index_cache_capacity_bytes->set_value(config.lance_index_cache_size_bytes); + lance_session_metadata_cache_capacity_bytes->set_value( + config.lance_metadata_cache_size_bytes); + _entity->register_hook(std::string(LANCE_SESSION_CACHE_METRICS_HOOK), + [this]() { update(); }); + update(); + } + + ~LanceSessionMetrics() { + _entity->deregister_hook(std::string(LANCE_SESSION_CACHE_METRICS_HOOK)); + METRIC_DEREGISTER(_entity, lance_session_index_cache_capacity_bytes); + METRIC_DEREGISTER(_entity, lance_session_index_cache_usage_bytes); + METRIC_DEREGISTER(_entity, lance_session_index_cache_entries); + METRIC_DEREGISTER(_entity, lance_session_index_cache_hits_total); + METRIC_DEREGISTER(_entity, lance_session_index_cache_misses_total); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_capacity_bytes); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_usage_bytes); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_entries); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_hits_total); + METRIC_DEREGISTER(_entity, lance_session_metadata_cache_misses_total); + } + +private: + void update() { + // Session caches are shared across queries, so publish one process-wide snapshot instead + // of attributing concurrent cache activity to an individual query profile. + LanceSessionCacheStats stats {}; + if (lance_session_get_cache_stats(_session, &stats) != 0) { + LOG_EVERY_N(WARNING, 100) + << lance_error("collect Lance session cache statistics").to_string(); + return; + } + lance_session_index_cache_usage_bytes->set_value( + metric_value(stats.index_cache_size_bytes)); + lance_session_index_cache_entries->set_value(metric_value(stats.index_cache_entries)); + lance_session_index_cache_hits_total->set_value(metric_value(stats.index_cache_hits)); + lance_session_index_cache_misses_total->set_value(metric_value(stats.index_cache_misses)); + lance_session_metadata_cache_usage_bytes->set_value( + metric_value(stats.metadata_cache_size_bytes)); + lance_session_metadata_cache_entries->set_value(metric_value(stats.metadata_cache_entries)); + lance_session_metadata_cache_hits_total->set_value(metric_value(stats.metadata_cache_hits)); + lance_session_metadata_cache_misses_total->set_value( + metric_value(stats.metadata_cache_misses)); + } + + LanceSession* _session; + MetricEntity* _entity; + IntGauge* lance_session_index_cache_capacity_bytes = nullptr; + IntGauge* lance_session_index_cache_usage_bytes = nullptr; + IntGauge* lance_session_index_cache_entries = nullptr; + IntCounter* lance_session_index_cache_hits_total = nullptr; + IntCounter* lance_session_index_cache_misses_total = nullptr; + IntGauge* lance_session_metadata_cache_capacity_bytes = nullptr; + IntGauge* lance_session_metadata_cache_usage_bytes = nullptr; + IntGauge* lance_session_metadata_cache_entries = nullptr; + IntCounter* lance_session_metadata_cache_hits_total = nullptr; + IntCounter* lance_session_metadata_cache_misses_total = nullptr; +}; + +LanceSessionManager& LanceSessionManager::instance() { + // Function-local static initialization is thread safe. Cache configuration is process scoped, + // so changing it requires a BE restart. + static LanceSessionManager manager(load_lance_session_config()); + return manager; +} + +LanceSessionManager::LanceSessionManager(Config config) : _config(std::move(config)) { + LOG(INFO) << "Creating BE-wide Lance session manager: lance_index_cache_size_bytes=" + << _config.lance_index_cache_size_bytes + << ", lance_metadata_cache_size_bytes=" << _config.lance_metadata_cache_size_bytes + << ", enable_lance_data_cache=" << _config.enable_lance_data_cache + << ", lance_data_cache_path=" << _config.lance_data_cache_path + << ", lance_data_cache_disk_capacity_bytes=" + << _config.lance_data_cache_disk_capacity_bytes + << ", lance_data_cache_read_block_size_bytes=" + << _config.lance_data_cache_read_block_size_bytes + << ", foyer_memory_capacity_bytes=" << _config.lance_data_cache_read_block_size_bytes; +} + +LanceSessionManager::~LanceSessionManager() { + _metrics.reset(); + lance_session_close(_session); +} + +Status LanceSessionManager::_initialize() { + if (_config.enable_lance_data_cache) { + // Treat the configured cache mode as a process-level requirement. If an enabled + // data cache cannot initialize, report the failure instead of silently creating a + // session without it. This keeps directory/configuration/device failures visible + // to the operator. Disabling the cache is an explicit configuration change. + const LanceDataCacheOptions data_cache_options { + .directory = _config.lance_data_cache_path.c_str(), + // Foyer's HybridCache requires a memory tier. Keep it at the minimum useful + // capacity of exactly one range-cache block; WriteOnInsertion enqueues disk + // writes on insertion rather than waiting for memory-tier eviction. + .memory_capacity_bytes = + static_cast(_config.lance_data_cache_read_block_size_bytes), + .disk_capacity_bytes = + static_cast(_config.lance_data_cache_disk_capacity_bytes), + .read_block_size_bytes = + static_cast(_config.lance_data_cache_read_block_size_bytes), + }; + _session = lance_session_new_with_data_cache( + static_cast(_config.lance_index_cache_size_bytes), + static_cast(_config.lance_metadata_cache_size_bytes), + &data_cache_options); + } else { + _session = + lance_session_new(static_cast(_config.lance_index_cache_size_bytes), + static_cast(_config.lance_metadata_cache_size_bytes)); + } + if (_session == nullptr) { + // Capture the Lance-C error on the initializing thread before another FFI call can + // replace it. Keep the original cause (including Foyer's directory/I/O details) and + // explain how to recover from the initialization status retained by call_once below. + auto status = lance_error("create shared Lance session"); + if (_config.enable_lance_data_cache) { + status.append( + "; Check the Lance data cache configuration and storage. After fixing the " + "issue, restart this BE. Alternatively, set enable_lance_data_cache=false " + "in be.conf and restart this BE. Session initialization will not be retried " + "in this BE process."); + } + return status; + } + _metrics = std::make_unique(_session, _config); + return Status::OK(); +} + +Status LanceSessionManager::open_dataset(const char* uri, const char* const* storage_options, + uint64_t version, LanceDataset** dataset) { + if (uri == nullptr || dataset == nullptr) { + return Status::InvalidArgument("Lance dataset URI and output must not be null"); + } + *dataset = nullptr; + + // Initialize lazily on the first dataset open, not at BE startup. A failed Status is a + // normal return from this lambda, so call_once completes and retains that failure just + // like a successful initialization. All subsequent readers receive the same copied + // error; fixing the cache directory alone does not trigger another attempt. This is + // intentional: repair the cache configuration/storage (or disable the data cache), then + // restart the BE to recreate the process-wide manager and session. + std::call_once(_initialize_once, [this] { _initialize_status = _initialize(); }); + RETURN_IF_ERROR(_initialize_status); + + *dataset = lance_dataset_open_with_session(uri, storage_options, version, _session); + if (*dataset == nullptr) { + return lance_error("open Lance dataset with shared session"); + } + return Status::OK(); +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_session_manager.h b/be/src/format_v2/lance/lance_session_manager.h new file mode 100644 index 00000000000000..45dc62acad4442 --- /dev/null +++ b/be/src/format_v2/lance/lance_session_manager.h @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" + +struct LanceDataset; +struct LanceSession; + +namespace doris::format::lance { + +class LanceSessionMetrics; + +// Owns the single Lance session shared by all queries in one BE process. The session always owns +// Lance's metadata/index caches and optionally installs the Foyer data-file cache. Readers only +// open datasets through this class and do not depend on the selected data-cache implementation. +class LanceSessionManager final { +public: + struct Config { + int64_t lance_index_cache_size_bytes = 0; + int64_t lance_metadata_cache_size_bytes = 0; + bool enable_lance_data_cache = false; + std::string lance_data_cache_path; + int64_t lance_data_cache_disk_capacity_bytes = 0; + int64_t lance_data_cache_read_block_size_bytes = 0; + }; + + static LanceSessionManager& instance(); + + // The explicit configuration constructor keeps the process-global config out of focused + // manager tests. Production readers use instance(). + explicit LanceSessionManager(Config config); + ~LanceSessionManager(); + + LanceSessionManager(const LanceSessionManager&) = delete; + LanceSessionManager& operator=(const LanceSessionManager&) = delete; + + Status open_dataset(const char* uri, const char* const* storage_options, uint64_t version, + LanceDataset** dataset); + +private: + Status _initialize(); + + Config _config; + std::once_flag _initialize_once; + LanceSession* _session = nullptr; + std::unique_ptr _metrics; + Status _initialize_status = Status::OK(); +}; + +} // namespace doris::format::lance diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 151744480eec95..1c9260d2aa3f1c 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -156,6 +156,8 @@ const char* file_format_name(FileFormat format) { return "ARROW"; case FileFormat::WAL: return "WAL"; + case FileFormat::LANCE: + return "LANCE"; } return "UNKNOWN"; } diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp new file mode 100644 index 00000000000000..dc5a233b13ab6e --- /dev/null +++ b/be/src/format_v2/table/lance_reader.cpp @@ -0,0 +1,1248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/lance_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "exec/common/endian.h" +#include "format_v2/lance/lance_reader_helper.h" +#include "format_v2/lance/lance_runtime_filter_helper.h" +#include "format_v2/lance/lance_session_manager.h" +#include "runtime/file_scan_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::lance { + +LanceTableReader::~LanceTableReader() { + static_cast(close()); +} + +Status LanceTableReader::fetch_schema(const TFileRangeDesc& range, + const TFileScanRangeParams& scan_params, + std::vector* column_names, + std::vector* column_types) const { + if (column_names == nullptr || column_types == nullptr) { + return Status::InvalidArgument("Lance schema output must not be null"); + } + const auto& params = range.table_format_params.lance_params; + std::vector storage_options; + RETURN_IF_ERROR(build_lance_storage_options(&scan_params, &storage_options)); + std::vector storage_option_ptrs; + storage_option_ptrs.reserve(storage_options.size() + 1); + for (const auto& option : storage_options) { + storage_option_ptrs.emplace_back(option.c_str()); + } + storage_option_ptrs.emplace_back(nullptr); + + std::unique_ptr dataset( + lance_dataset_open(params.dataset_uri.c_str(), + storage_options.empty() ? nullptr : storage_option_ptrs.data(), + static_cast(params.version))); + if (dataset == nullptr) { + return lance_error("open Lance dataset for schema"); + } + + std::shared_ptr schema; + RETURN_IF_ERROR(import_lance_dataset_schema(dataset.get(), &schema)); + return convert_arrow_schema_to_doris(schema, column_names, column_types); +} + +Status LanceTableReader::init(TableReadOptions&& options) { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + DORIS_CHECK(_runtime_state != nullptr); + DORIS_CHECK(_scanner_profile != nullptr); + DORIS_CHECK(_scan_params != nullptr); + RETURN_IF_ERROR(_resolve_search_kind()); + + const auto& lance_scan_params = _scan_params->lance_scan_params; + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, LANCE_READER_PROFILE, TUnit::NONE, + file_scan_profile::TABLE_READER, 1); + _dataset_open_time = add_lance_counter(_scanner_profile, "LanceDatasetOpenTime", TUnit::TIME_NS, + LANCE_TIMING_PROFILE); + _arrow_to_doris_block_time = add_lance_counter(_scanner_profile, "LanceArrowToDorisBlockTime", + TUnit::TIME_NS, LANCE_TIMING_PROFILE); + _data_cache_bytes_read_from_cache = + add_lance_counter(_scanner_profile, "LanceDataCacheBytesReadFromCache", TUnit::BYTES, + LANCE_DATA_CACHE_PROFILE); + _data_cache_bytes_read_from_remote = + add_lance_counter(_scanner_profile, "LanceDataCacheBytesReadFromRemote", TUnit::BYTES, + LANCE_DATA_CACHE_PROFILE); + if (_search_kind != SearchKind::NORMAL) { + RETURN_IF_ERROR(_validate_external_search_request()); + const auto& request = lance_scan_params.external_search_request; + int64_t top_k; + int64_t offset; + if (_search_kind == SearchKind::VECTOR) { + const auto& vector = request.search_query.vector_search; + top_k = vector.top_k; + offset = vector.offset; + _scanner_profile->add_info_string("LanceSearchType", "VECTOR"); + } else { + DORIS_CHECK(_search_kind == SearchKind::FULL_TEXT); + const auto& full_text = request.search_query.full_text_search; + top_k = full_text.top_k; + offset = full_text.offset; + _scanner_profile->add_info_string("LanceSearchType", "FULL_TEXT"); + _scanner_profile->add_info_string( + "LanceFtsCoverageMode", + full_text.coverage_mode == TFtsCoverageMode::STRICT ? "STRICT" : "INDEX_ONLY"); + if (full_text.query_type == TFtsQueryType::MATCH) { + _scanner_profile->add_info_string("LanceFtsQueryType", "MATCH"); + _scanner_profile->add_info_string( + "LanceFtsMatchOperator", + full_text.match_operator == TFtsMatchOperator::AND ? "AND" : "OR"); + _scanner_profile->add_info_string("LanceFtsMaxFuzzyDistance", + std::to_string(full_text.max_fuzzy_distance)); + } else { + _scanner_profile->add_info_string("LanceFtsQueryType", "PHRASE"); + _scanner_profile->add_info_string("LanceFtsPhraseSlop", + std::to_string(full_text.phrase_slop)); + } + } + _scanner_profile->add_info_string("LanceTopK", std::to_string(top_k)); + _scanner_profile->add_info_string("LanceOffset", std::to_string(offset)); + } + if (_scan_params->__isset.lance_scan_params && + lance_scan_params.__isset.lance_substrait_filter) { + _scanner_profile->add_info_string("LancePushdownFormat", "SUBSTRAIT"); + } + + return _record_batch_converter.init(_runtime_state, _projected_columns, _search_kind); +} + +Status LanceTableReader::prepare_split(const SplitReadOptions& options) { + _close_scanner(); + _eof = false; + _runtime_filter_cache = options.cache; + + RETURN_IF_ERROR(TableReader::prepare_split(options)); + if (current_split_pruned()) { + return Status::OK(); + } + // COUNT(*)/COUNT(1) with no filter is served from Lance metadata. The base class already set + // _remaining_table_level_count from the split's table_level_row_count, so skip opening any + // dataset scanner; get_block() synthesizes the counted rows. + if (_is_table_level_count_active()) { + return Status::OK(); + } + if (_record_batch_converter.requires_global_rowid() && !_global_rowid_context.has_value()) { + return Status::InvalidArgument( + "Lance global row id requested without global row id context"); + } + + RETURN_IF_ERROR(_ensure_dataset_open(options.current_range)); + RETURN_IF_ERROR(_open_scanner(options.current_range)); + return Status::OK(); +} + +Status LanceTableReader::get_block(Block* block, bool* eos) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(eos != nullptr); + DORIS_CHECK(block->columns() == _projected_columns.size()); + *eos = false; + + if (_eof) { + *eos = true; + return Status::OK(); + } + // Metadata COUNT(*) split: no scanner is opened. Emit synthetic rows for the upper COUNT + // operator directly from the row count the base class parsed out of the split. + if (_is_table_level_count_active()) { + return _read_table_level_count(block, eos); + } + if (_scanner == nullptr) { + return Status::InternalError("Lance scanner is not initialized for the current split"); + } + + const auto target_rows = std::max(1, _scanner_batch_size); + while (true) { + block->clear_column_data(_projected_columns.size()); + size_t raw_rows = 0; + while (raw_rows < target_rows) { + if (_io_ctx != nullptr && _io_ctx->should_stop) { + _eof = true; + _close_scanner(); + *eos = true; + return Status::OK(); + } + + LanceBatch* raw_batch = nullptr; + int32_t scan_status = 0; + { + SCOPED_TIMER(_scanner_next_time); + scan_status = lance_scanner_next(_scanner, &raw_batch); + } + if (scan_status == 1) { + _eof = true; + _close_scanner(); + break; + } + if (scan_status != 0 || raw_batch == nullptr) { + return lance_error("read next Lance batch"); + } + + std::unique_ptr batch(raw_batch); + size_t rows = 0; + { + SCOPED_TIMER(_arrow_to_doris_block_time); + RETURN_IF_ERROR(_fill_block_from_lance_batch(batch.get(), block, &rows)); + } + _record_scan_rows(rows); + raw_rows += rows; + } + + if (raw_rows == 0) { + DORIS_CHECK(_eof); + *eos = true; + return Status::OK(); + } + + if (block->rows() > 0) { + // Preserve a non-empty final block. The next get_block() observes `_eof` and reports + // split EOF, matching the generic reader contract. + return Status::OK(); + } + if (_eof) { + *eos = true; + return Status::OK(); + } + } +} + +Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range, + const std::vector& row_ids, Block* block) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(block->columns() == _projected_columns.size()); + if (row_ids.empty()) { + return Status::OK(); + } + if (_row_id_take_read_time == nullptr) { + _row_id_take_read_time = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "LanceRowIdTakeReadTime", LANCE_TIMING_PROFILE, 1); + } + if (_row_id_fetch_total_time == nullptr) { + _row_id_fetch_total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "LanceRowIdFetchTotalTime", LANCE_TIMING_PROFILE, 1); + } + auto* fetch_calls = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceRowIdFetchCalls", + TUnit::UNIT, LANCE_READER_PROFILE, 1); + auto* fetch_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceRowIdFetchRows", + TUnit::UNIT, LANCE_READER_PROFILE, 1); + SCOPED_TIMER(_row_id_fetch_total_time); + + // Phase-two row fetch does not execute FTS, so a reader created only for take_rows must not + // collect query-specific global statistics. + RETURN_IF_ERROR(_ensure_dataset_open(range, false)); + std::vector columns; + columns.reserve(_projected_columns.size() + 1); + for (const auto& column : _projected_columns) { + columns.emplace_back(column.name.c_str()); + } + columns.emplace_back(nullptr); + + ArrowArrayStream stream {}; + int32_t take_rows_status = 0; + { + SCOPED_TIMER(_row_id_take_read_time); + COUNTER_UPDATE(fetch_calls, 1); + take_rows_status = lance_dataset_take_rows(_dataset, row_ids.data(), row_ids.size(), + columns.data(), &stream); + } + if (take_rows_status != 0) { + if (stream.release != nullptr) { + stream.release(&stream); + } + return lance_error("take Lance rows by row id"); + } + auto imported_reader = arrow::ImportRecordBatchReader(&stream); + if (!imported_reader.ok()) { + if (stream.release != nullptr) { + stream.release(&stream); + } + return Status::InternalError("import Lance take-rows stream failed: {}", + imported_reader.status().message()); + } + + size_t fetched_rows = 0; + auto batch_reader = std::move(imported_reader).ValueUnsafe(); + while (true) { + std::shared_ptr record_batch; + arrow::Status read_status; + { + // Lance may materialize take_rows lazily while its Arrow stream is consumed. + SCOPED_TIMER(_row_id_take_read_time); + read_status = batch_reader->ReadNext(&record_batch); + } + if (!read_status.ok()) { + return Status::InternalError("read Lance take-rows batch failed: {}", + read_status.message()); + } + if (record_batch == nullptr) { + break; + } + size_t rows = 0; + { + SCOPED_TIMER(_arrow_to_doris_block_time); + RETURN_IF_ERROR(_record_batch_converter.convert_record_batch_to_block( + record_batch, block, _global_rowid_context, &rows)); + } + fetched_rows += rows; + COUNTER_UPDATE(fetch_rows, rows); + } + if (fetched_rows != row_ids.size()) { + return Status::InternalError("Lance row-id fetch returned {} rows for {} requested row ids", + fetched_rows, row_ids.size()); + } + return Status::OK(); +} + +Status LanceTableReader::abort_split() { + _close_scanner(); + _eof = true; + return TableReader::abort_split(); +} + +Status LanceTableReader::close() { + _close_scanner(); + _close_dataset(); + _opened_dataset_key.reset(); + _eof = true; + return TableReader::close(); +} + +Status LanceTableReader::_resolve_search_kind() { + DORIS_CHECK(_scan_params != nullptr); + _search_kind = SearchKind::NORMAL; + if (!_scan_params->__isset.lance_scan_params) { + return Status::OK(); + } + const auto& lance_scan_params = _scan_params->lance_scan_params; + if (!lance_scan_params.__isset.external_search_request) { + return Status::OK(); + } + const auto& request = lance_scan_params.external_search_request; + if (!request.__isset.search_query) { + return Status::InvalidArgument("external search request requires search_query"); + } + const bool has_vector = request.search_query.__isset.vector_search; + const bool has_full_text = request.search_query.__isset.full_text_search; + if (has_vector == has_full_text) { + return Status::InvalidArgument("external search query must set exactly one search kind"); + } + _search_kind = has_vector ? SearchKind::VECTOR : SearchKind::FULL_TEXT; + return Status::OK(); +} + +Status LanceTableReader::_validate_external_search_request() const { + // FE validates requests produced by the search TVFs, but this reader consumes a deserialized + // Thrift boundary. Recheck structural invariants and values used for allocation, pointer + // arithmetic, C-string calls, and narrowing conversions before accessing them below. + DORIS_CHECK(_scan_params != nullptr); + DORIS_CHECK(_scan_params->__isset.lance_scan_params); + const auto& lance_scan_params = _scan_params->lance_scan_params; + DORIS_CHECK(lance_scan_params.__isset.external_search_request); + if (lance_scan_params.__isset.lance_substrait_filter) { + return Status::InvalidArgument( + "Lance external search cannot combine external_search_request with " + "lance_substrait_filter"); + } + + const auto& request = lance_scan_params.external_search_request; + if (request.schema_version != 1) { + return Status::NotSupported("unsupported external search schema version: {}", + request.schema_version); + } + DORIS_CHECK(request.__isset.search_query); + DORIS_CHECK(_search_kind != SearchKind::NORMAL); + constexpr auto UINT32_MAX_VALUE = static_cast(std::numeric_limits::max()); + if (_search_kind == SearchKind::VECTOR) { + const auto& vector = request.search_query.vector_search; + if (!vector.__isset.column || vector.column.empty() || + vector.column.find('\0') != std::string::npos) { + return Status::InvalidArgument("Lance vector search requires a non-empty column"); + } + if (!vector.__isset.query_vector) { + return Status::InvalidArgument("Lance vector search requires a query vector"); + } + const auto& query_vector = vector.query_vector; + if (!query_vector.__isset.element_type || !query_vector.__isset.dimension || + !query_vector.__isset.values) { + return Status::InvalidArgument( + "Lance query vector requires element_type, dimension, and values"); + } + if (query_vector.dimension <= 0) { + return Status::InvalidArgument("Lance query vector dimension must be positive: {}", + query_vector.dimension); + } + const auto element_width = lance_vector_element_width(query_vector.element_type); + if (element_width == 0) { + return Status::NotSupported("unsupported Lance query vector element type: {}", + static_cast(query_vector.element_type)); + } + const bool multi_vector = query_vector.__isset.num_vectors; + const bool has_distance_range = + vector.__isset.distance_lower_bound || vector.__isset.distance_upper_bound; + if (has_distance_range && multi_vector) { + return Status::NotSupported( + "Distance bounds are not supported for Lance multi-vector search"); + } + // Validate doubles before narrowing: non-finite or out-of-range values cannot reach the + // C API. + const auto valid_bound = [](double bound) { + return std::isfinite(bound) && std::abs(bound) <= std::numeric_limits::max(); + }; + if ((vector.__isset.distance_lower_bound && !valid_bound(vector.distance_lower_bound)) || + (vector.__isset.distance_upper_bound && !valid_bound(vector.distance_upper_bound))) { + return Status::InvalidArgument("Lance distance bounds must be finite FLOAT values"); + } + if (vector.__isset.distance_lower_bound && vector.__isset.distance_upper_bound && + static_cast(vector.distance_lower_bound) >= + static_cast(vector.distance_upper_bound)) { + return Status::InvalidArgument( + "distance_lower_bound must be less than distance_upper_bound at FLOAT " + "precision"); + } + + // The optional count distinguishes a query matrix, including a one-row matrix. + if (multi_vector && query_vector.num_vectors <= 0) { + return Status::InvalidArgument( + "Lance multi-vector queries require positive num_vectors"); + } + if (multi_vector && (query_vector.element_type == TVectorElementType::UINT8 || + query_vector.element_type == TVectorElementType::INT8 || + (vector.__isset.metric && vector.metric == TVectorMetric::HAMMING))) { + return Status::NotSupported( + "Lance multi-vector search requires floating-point vectors and l2, cosine, or " + "dot"); + } + const auto dimension = static_cast(query_vector.dimension); + const auto count = multi_vector ? static_cast(query_vector.num_vectors) : 1; + if (dimension > std::numeric_limits::max() / count / element_width || + query_vector.values.size() != dimension * count * element_width) { + return Status::InvalidArgument( + "Lance query vector byte size {} does not match {} vectors of dimension {} and " + "element width {}", + query_vector.values.size(), count, dimension, element_width); + } + if (!vector.__isset.top_k || vector.top_k <= 0) { + return Status::InvalidArgument("Lance vector search top_k must be positive"); + } + if (!vector.__isset.offset || vector.offset < 0) { + return Status::InvalidArgument("Lance vector search offset must be non-negative"); + } + if (vector.offset > UINT32_MAX_VALUE || vector.top_k > UINT32_MAX_VALUE - vector.offset) { + return Status::InvalidArgument( + "Lance vector search top_k + offset exceeds uint32 range"); + } + // Match FE/C API limits before constructing the per-subvector ANN plan branches. + constexpr int MAX_QUERY_VECTORS = 128; + constexpr int64_t MAX_QUERY_VECTOR_CANDIDATES = 100000; + const auto refine_factor = + request.__isset.vector_search_options && + request.vector_search_options.__isset.refine_factor + ? request.vector_search_options.refine_factor + : 1; + if (multi_vector && + (query_vector.num_vectors > MAX_QUERY_VECTORS || refine_factor <= 0 || + vector.top_k + vector.offset > MAX_QUERY_VECTOR_CANDIDATES / refine_factor || + query_vector.num_vectors > + MAX_QUERY_VECTOR_CANDIDATES / (vector.top_k + vector.offset))) { + return Status::InvalidArgument( + "multi-vector query exceeds 128 subvectors or 100000 subvector-candidates"); + } + } else { + DORIS_CHECK(_search_kind == SearchKind::FULL_TEXT); + const auto& full_text = request.search_query.full_text_search; + if (!full_text.__isset.column || full_text.column.empty() || + full_text.column.find('\0') != std::string::npos) { + return Status::InvalidArgument("Lance full-text search requires a non-empty column"); + } + if (!full_text.__isset.query || full_text.query.empty() || + full_text.query.find('\0') != std::string::npos) { + return Status::InvalidArgument("Lance full-text search requires a non-empty query"); + } + if (!full_text.__isset.top_k || full_text.top_k <= 0) { + return Status::InvalidArgument("Lance full-text search top_k must be positive"); + } + if (!full_text.__isset.offset || full_text.offset < 0) { + return Status::InvalidArgument("Lance full-text search offset must be non-negative"); + } + if (full_text.offset > UINT32_MAX_VALUE || + full_text.top_k > UINT32_MAX_VALUE - full_text.offset) { + return Status::InvalidArgument( + "Lance full-text search top_k + offset exceeds uint32 range"); + } + if (!full_text.__isset.coverage_mode || + (full_text.coverage_mode != TFtsCoverageMode::STRICT && + full_text.coverage_mode != TFtsCoverageMode::INDEX_ONLY)) { + return Status::InvalidArgument( + "Lance full-text search requires STRICT or INDEX_ONLY coverage_mode"); + } + if (full_text.__isset.global_statistics && full_text.global_statistics.empty()) { + return Status::InvalidArgument( + "Lance full-text search global_statistics must not be empty when set"); + } + if (!full_text.__isset.query_type || (full_text.query_type != TFtsQueryType::MATCH && + full_text.query_type != TFtsQueryType::PHRASE)) { + return Status::InvalidArgument( + "Lance full-text search requires MATCH or PHRASE query_type"); + } + if (full_text.query_type == TFtsQueryType::MATCH) { + if (!full_text.__isset.match_operator || + (full_text.match_operator != TFtsMatchOperator::OR && + full_text.match_operator != TFtsMatchOperator::AND)) { + return Status::InvalidArgument( + "Lance MATCH query requires OR or AND match_operator"); + } + if (!full_text.__isset.max_fuzzy_distance || full_text.max_fuzzy_distance < 0) { + return Status::InvalidArgument( + "Lance MATCH query max_fuzzy_distance must be non-negative"); + } + if (full_text.max_fuzzy_distance != 0) { + return Status::NotSupported( + "Lance prepared FTS does not yet support max_fuzzy_distance={}", + full_text.max_fuzzy_distance); + } + if (full_text.__isset.phrase_slop) { + return Status::InvalidArgument("Lance MATCH query cannot set phrase_slop"); + } + } else { + if (!full_text.__isset.phrase_slop || full_text.phrase_slop < 0) { + return Status::InvalidArgument( + "Lance PHRASE query phrase_slop must be non-negative"); + } + if (full_text.__isset.match_operator || full_text.__isset.max_fuzzy_distance) { + return Status::InvalidArgument( + "Lance PHRASE query cannot set MATCH-only parameters"); + } + } + if (request.__isset.vector_search_options) { + return Status::InvalidArgument( + "Lance full-text search cannot set vector_search_options"); + } + } + + if (request.__isset.search_filter) { + const auto& filter = request.search_filter; + if (!filter.__isset.format || !filter.__isset.payload || filter.payload.empty()) { + return Status::InvalidArgument( + "external search filter requires format and non-empty payload"); + } + if (filter.format != TSearchFilterFormat::SQL) { + return Status::NotSupported("unsupported external search filter format: {}", + static_cast(filter.format)); + } + if (filter.payload.find('\0') != std::string::npos) { + return Status::InvalidArgument("Lance SQL search filter contains an embedded NUL byte"); + } + } + + if (_search_kind == SearchKind::VECTOR && request.__isset.vector_search_options) { + const auto& options = request.vector_search_options; + if (options.__isset.query_parallelism && options.query_parallelism < -1) { + return Status::InvalidArgument("Lance query_parallelism must be -1, 0, or positive"); + } + if (options.__isset.nprobes && options.nprobes <= 0) { + return Status::InvalidArgument("Lance nprobes must be positive"); + } + if (options.__isset.refine_factor && options.refine_factor <= 0) { + return Status::InvalidArgument("Lance refine_factor must be positive"); + } + if (options.__isset.ef && options.ef <= 0) { + return Status::InvalidArgument("Lance ef must be positive"); + } + } + return Status::OK(); +} + +Status LanceTableReader::_ensure_dataset_open(const TFileRangeDesc& range, + bool prepare_fts_context) { + DatasetKey key; + RETURN_IF_ERROR(_dataset_key(range, &key)); + if (_dataset == nullptr) { + RETURN_IF_ERROR(_open_dataset(key)); + _opened_dataset_key = key; + } else if (!_opened_dataset_key.has_value() || *_opened_dataset_key != key) { + return Status::InvalidArgument( + "Lance reader cannot mix dataset snapshots or storage options"); + } + if (_search_kind == SearchKind::FULL_TEXT && prepare_fts_context && + _fts_query_context == nullptr) { + RETURN_IF_ERROR(_prepare_fts_query_context()); + } + return Status::OK(); +} + +Status LanceTableReader::_open_dataset(const DatasetKey& key) { + std::vector storage_option_ptrs; + storage_option_ptrs.reserve(key.storage_options.size() + 1); + for (const auto& option : key.storage_options) { + storage_option_ptrs.emplace_back(option.c_str()); + } + storage_option_ptrs.emplace_back(nullptr); + + std::unique_ptr dataset; + { + SCOPED_TIMER(_dataset_open_time); + LanceDataset* raw_dataset = nullptr; + RETURN_IF_ERROR(LanceSessionManager::instance().open_dataset( + key.uri.c_str(), key.storage_options.empty() ? nullptr : storage_option_ptrs.data(), + static_cast(key.version), &raw_dataset)); + dataset.reset(raw_dataset); + } + _dataset = dataset.release(); + return Status::OK(); +} + +Status LanceTableReader::_prepare_fts_query_context() { + DORIS_CHECK(_dataset != nullptr); + DORIS_CHECK(_fts_query_context == nullptr); + DORIS_CHECK(_scan_params != nullptr); + const auto& full_text = + _scan_params->lance_scan_params.external_search_request.search_query.full_text_search; + if (full_text.__isset.global_statistics) { + return Status::NotSupported( + "Lance FE-provided FTS global statistics require a lance-c consumer API"); + } + const auto coverage_mode = full_text.coverage_mode == TFtsCoverageMode::STRICT + ? LANCE_FTS_COVERAGE_STRICT + : LANCE_FTS_COVERAGE_INDEX_ONLY; + // Keep statistics preparation at the reader/scanner lifetime today. A future FE-provided + // opaque statistics payload should enter through this boundary and create the same context, + // leaving segment-scoped scanner execution unchanged. + if (full_text.query_type == TFtsQueryType::MATCH) { + const auto match_operator = full_text.match_operator == TFtsMatchOperator::AND + ? LANCE_FTS_MATCH_OPERATOR_AND + : LANCE_FTS_MATCH_OPERATOR_OR; + _fts_query_context = lance_dataset_prepare_fts_match_query( + _dataset, full_text.column.c_str(), full_text.query.c_str(), match_operator, + static_cast(full_text.max_fuzzy_distance), coverage_mode); + } else { + DORIS_CHECK(full_text.query_type == TFtsQueryType::PHRASE); + _fts_query_context = lance_dataset_prepare_fts_phrase_query( + _dataset, full_text.column.c_str(), full_text.query.c_str(), full_text.phrase_slop, + coverage_mode); + } + if (_fts_query_context == nullptr) { + return lance_error("prepare Lance FTS query context"); + } + return Status::OK(); +} + +void LanceTableReader::_init_scanner_profile() { + if (_scanner_configure_time != nullptr) { + return; + } + + _scanner_configure_time = add_lance_counter(_scanner_profile, "LanceScannerConfigureTime", + TUnit::TIME_NS, LANCE_TIMING_PROFILE); + _runtime_filter_sql_translation_time = + add_lance_counter(_scanner_profile, "LanceRuntimeFilterSqlTranslationTime", + TUnit::TIME_NS, LANCE_TIMING_PROFILE); + _scanner_next_time = add_lance_counter(_scanner_profile, "LanceScannerNextTime", TUnit::TIME_NS, + LANCE_TIMING_PROFILE); + _execution_iops = + add_lance_counter(_scanner_profile, "LanceIOReadOps", TUnit::UNIT, LANCE_IO_PROFILE); + _execution_requests = add_lance_counter(_scanner_profile, "LanceIOReadRequests", TUnit::UNIT, + LANCE_IO_PROFILE); + _execution_bytes_read = + add_lance_counter(_scanner_profile, "LanceIOReadBytes", TUnit::BYTES, LANCE_IO_PROFILE); + _index_object_loads = add_lance_counter(_scanner_profile, "LanceIndexObjectLoads", TUnit::UNIT, + LANCE_INDEX_PROFILE); + _index_components_loaded = add_lance_counter(_scanner_profile, "LanceIndexComponentsLoaded", + TUnit::UNIT, LANCE_INDEX_PROFILE); + _index_comparisons = add_lance_counter(_scanner_profile, "LanceIndexComparisons", TUnit::UNIT, + LANCE_INDEX_PROFILE); + + _scan_metric_counters.reserve(sizeof(LANCE_SCAN_METRICS) / sizeof(LANCE_SCAN_METRICS[0])); + for (const auto& definition : LANCE_SCAN_METRICS) { + _scan_metric_counters.emplace_back(add_lance_counter( + _scanner_profile, definition.profile_name, definition.unit, definition.group)); + } + + if (_search_kind != SearchKind::NORMAL) { + _planned_index_segment_count = + add_lance_counter(_scanner_profile, "LancePlannedIndexSegmentCount", TUnit::UNIT, + LANCE_SEARCH_PLAN_PROFILE); + _planned_indexed_fragment_count = + add_lance_counter(_scanner_profile, "LancePlannedIndexedFragmentCount", TUnit::UNIT, + LANCE_SEARCH_PLAN_PROFILE); + if (_search_kind == SearchKind::VECTOR) { + _planned_flat_search_fragment_count = + add_lance_counter(_scanner_profile, "LancePlannedFlatSearchFragmentCount", + TUnit::UNIT, LANCE_SEARCH_PLAN_PROFILE); + } + } +} + +Status LanceTableReader::_open_scanner(const TFileRangeDesc& range) { + _init_scanner_profile(); + SCOPED_TIMER(_scanner_configure_time); + std::vector columns; + columns.reserve(_projected_columns.size() + 1); + for (size_t idx = 0; idx < _projected_columns.size(); ++idx) { + if (_record_batch_converter.is_global_rowid_output(idx)) { + continue; + } + const auto& column = _projected_columns[idx]; + columns.emplace_back(column.name.c_str()); + } + if (_search_kind != SearchKind::NORMAL && columns.empty()) { + // Keep an explicit empty user projection from becoming `nullptr`, which means all dataset + // columns to lance-c. Search execution already returns its generated result column. + columns.emplace_back(_search_kind == SearchKind::VECTOR ? LANCE_DISTANCE_COLUMN.data() + : LANCE_SCORE_COLUMN.data()); + } + columns.emplace_back(nullptr); + + const auto& lance_scan_params = _scan_params->lance_scan_params; + std::string sql_filter; + std::shared_ptr runtime_filter_sql; + if (_search_kind == SearchKind::NORMAL) { + if (has_lance_runtime_filters(_conjuncts)) { + if (_dataset_schema == nullptr) { + RETURN_IF_ERROR(import_lance_dataset_schema(_dataset, &_dataset_schema)); + } + SCOPED_TIMER(_runtime_filter_sql_translation_time); + runtime_filter_sql = get_or_create_lance_runtime_filter_sql( + _conjuncts, *_dataset_schema, _runtime_filter_cache); + } + } else { + const auto& request = lance_scan_params.external_search_request; + if (request.__isset.search_filter && + request.search_filter.format == TSearchFilterFormat::SQL) { + sql_filter = request.search_filter.payload; + } + } + LanceScanner* scanner = lance_scanner_new(_dataset, columns.data(), + sql_filter.empty() ? nullptr : sql_filter.c_str()); + if (scanner == nullptr) { + return lance_error("create Lance scanner"); + } + std::unique_ptr scanner_guard(scanner); + const auto collect_scan_statistics = [](void* callback_ctx, + const LanceScanStatistics* statistics) noexcept { + try { + LanceTableReader::_collect_scan_statistics(callback_ctx, statistics); + } catch (...) { + // Lance statistics are best-effort; never unwind through the callback ABI. + } + }; + if (lance_scanner_set_statistics_callback(scanner, collect_scan_statistics, this) != 0) { + return lance_error("set Lance scanner statistics callback"); + } + + if (_record_batch_converter.requires_global_rowid() && + lance_scanner_with_row_id(scanner, true) != 0) { + return lance_error("enable Lance row id output"); + } + + if (lance_scan_params.__isset.lance_substrait_filter && + lance_scanner_set_substrait_filter( + scanner, + reinterpret_cast(lance_scan_params.lance_substrait_filter.data()), + lance_scan_params.lance_substrait_filter.size()) != 0) { + return lance_error("set Lance Substrait filter"); + } + if (runtime_filter_sql != nullptr) { + if (!runtime_filter_sql->expression.empty()) { + if (lance_scanner_additional_sql_filter(scanner, + runtime_filter_sql->expression.c_str()) != 0) { + return lance_error("set Lance additional SQL filter"); + } + } + record_lance_runtime_filter_pushdown(_scanner_profile, *runtime_filter_sql); + } + + const auto batch_size = _batch_size > 0 ? _batch_size : _runtime_state->batch_size(); + if (lance_scanner_set_batch_size(scanner, static_cast(batch_size)) != 0) { + return lance_error("set Lance scanner batch size"); + } + RETURN_IF_ERROR(_configure_scan_options(scanner)); + + const auto& lance_params = range.table_format_params.lance_params; + switch (_search_kind) { + case SearchKind::NORMAL: + RETURN_IF_ERROR(_configure_normal_scan(scanner, lance_params)); + break; + case SearchKind::VECTOR: + RETURN_IF_ERROR(_configure_vector_search(scanner, lance_params)); + break; + case SearchKind::FULL_TEXT: + RETURN_IF_ERROR(_configure_full_text_search(scanner, lance_params)); + break; + } + _scanner = scanner_guard.release(); + _scanner_batch_size = batch_size; + return Status::OK(); +} + +Status LanceTableReader::_configure_scan_options(LanceScanner* scanner) const { + DORIS_CHECK(scanner != nullptr); + // Doris runs multiple scanners concurrently. Limit each scanner's read-ahead; + // the I/O budget does not cap its total memory usage. + const auto io_buffer_size = static_cast(config::lance_io_buffer_size_bytes); + const auto batch_readahead = static_cast(config::lance_batch_readahead); + const auto fragment_readahead = static_cast(config::lance_fragment_readahead); + constexpr bool scan_in_order = false; + + if (lance_scanner_set_io_buffer_size(scanner, io_buffer_size) != 0) { + return lance_error("set Lance scanner I/O buffer size"); + } + if (lance_scanner_set_batch_readahead(scanner, batch_readahead) != 0) { + return lance_error("set Lance scanner batch readahead"); + } + if (lance_scanner_set_fragment_readahead(scanner, fragment_readahead) != 0) { + return lance_error("set Lance scanner fragment readahead"); + } + // Storage order is not required; query ordering is enforced by Sort/TopN operators. + if (lance_scanner_set_scan_in_order(scanner, scan_in_order) != 0) { + return lance_error("set Lance scanner scan order"); + } + + return Status::OK(); +} + +Status LanceTableReader::_configure_normal_scan(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const { + DORIS_CHECK(scanner != nullptr); + std::vector fragment_ids; + RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids)); + if (!fragment_ids.empty() && + lance_scanner_set_fragment_ids(scanner, fragment_ids.data(), fragment_ids.size()) != 0) { + return lance_error("set Lance scanner fragment ids"); + } + std::vector segment_uuids; + size_t segment_count = 0; + RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count)); + if (segment_count > 1) { + return Status::InvalidArgument("normal Lance scan accepts only one scalar index segment"); + } + if (segment_count == 1) { + if (fragment_ids.empty() || !lance_params.__isset.version || lance_params.version <= 0) { + return Status::InvalidArgument( + "Lance scalar index segment requires a fixed version and nonempty fragment " + "ids"); + } + if (lance_params.__isset.use_scalar_index && !lance_params.use_scalar_index) { + return Status::InvalidArgument( + "Lance scalar index segment cannot be combined with use_scalar_index=false"); + } + if (lance_scanner_set_scalar_index_segment(scanner, segment_uuids.data()) != 0) { + return lance_error("set Lance scanner scalar index segment"); + } + } else if (lance_params.__isset.use_scalar_index && + lance_scanner_set_use_scalar_index(scanner, lance_params.use_scalar_index) != 0) { + return lance_error("set Lance scanner scalar index usage"); + } + // FE sets this only when every predicate has been pushed into Lance. + if (lance_params.__isset.limit && lance_params.limit > 0 && + lance_scanner_set_limit(scanner, lance_params.limit) != 0) { + return lance_error("set Lance scanner limit"); + } + return Status::OK(); +} + +Status LanceTableReader::_configure_vector_search(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const { + DORIS_CHECK(scanner != nullptr); + DORIS_CHECK(_scan_params != nullptr); + DORIS_CHECK(_scan_params->__isset.lance_scan_params); + const auto& lance_scan_params = _scan_params->lance_scan_params; + DORIS_CHECK(lance_scan_params.__isset.external_search_request); + const auto& request = lance_scan_params.external_search_request; + const bool prefilter = request.__isset.prefilter && request.prefilter; + std::vector fragment_ids; + RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids)); + if (!fragment_ids.empty() && + lance_scanner_set_fragment_ids(scanner, fragment_ids.data(), fragment_ids.size()) != 0) { + return lance_error("set Lance vector scanner fragment ids"); + } + std::vector segment_uuids; + size_t segment_count = 0; + RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count)); + // FE plans index segments only when use_index allows them. A split carrying both contradicts + // itself, and the BE would have to override either FE's plan or the request. + if (segment_count > 0 && request.__isset.vector_search_options && + request.vector_search_options.__isset.use_index && + !request.vector_search_options.use_index) { + return Status::InvalidArgument( + "Lance vector search split carries {} index segments although use_index is false", + segment_count); + } + if (segment_count > 0 && + lance_scanner_set_index_segments(scanner, segment_uuids.data(), segment_count) != 0) { + return lance_error("set Lance vector scanner index segments"); + } + // This setting controls only the external-search TVF's search_filter. Doris WHERE conjuncts + // remain scanner residuals and are not included in external_search_request. + if (lance_scanner_set_prefilter(scanner, prefilter) != 0) { + return lance_error("set Lance vector search prefilter"); + } + const auto& vector = request.search_query.vector_search; + const auto& query = vector.query_vector; + const auto dimension = static_cast(query.dimension); + const auto count = query.__isset.num_vectors ? static_cast(query.num_vectors) : 1; + const auto num_elements = dimension * count; + const auto* bytes = query.values.data(); + const auto candidate_k = static_cast(vector.top_k + vector.offset); + + const auto set_nearest = [&](const void* values, LanceDataType type) -> Status { + const int result = + query.__isset.num_vectors + ? lance_scanner_nearest_multivector(scanner, vector.column.c_str(), values, + dimension, count, type, candidate_k) + : lance_scanner_nearest(scanner, vector.column.c_str(), values, dimension, + type, candidate_k); + if (result != 0) { + return lance_error("set Lance nearest query"); + } + return Status::OK(); + }; + + switch (query.element_type) { + case TVectorElementType::FLOAT16: { + std::vector values(num_elements); + for (size_t i = 0; i < num_elements; ++i) { + values[i] = LittleEndian::Load16(bytes + i * sizeof(uint16_t)); + } + RETURN_IF_ERROR(set_nearest(values.data(), LANCE_DTYPE_FLOAT16)); + break; + } + case TVectorElementType::FLOAT32: { + std::vector values(num_elements); + for (size_t i = 0; i < num_elements; ++i) { + const auto bits = LittleEndian::Load32(bytes + i * sizeof(uint32_t)); + values[i] = std::bit_cast(bits); + } + RETURN_IF_ERROR(set_nearest(values.data(), LANCE_DTYPE_FLOAT32)); + break; + } + case TVectorElementType::FLOAT64: { + std::vector values(num_elements); + for (size_t i = 0; i < num_elements; ++i) { + const auto bits = LittleEndian::Load64(bytes + i * sizeof(uint64_t)); + values[i] = std::bit_cast(bits); + } + RETURN_IF_ERROR(set_nearest(values.data(), LANCE_DTYPE_FLOAT64)); + break; + } + case TVectorElementType::UINT8: { + std::vector values(num_elements); + std::memcpy(values.data(), bytes, num_elements); + RETURN_IF_ERROR(set_nearest(values.data(), LANCE_DTYPE_UINT8)); + break; + } + case TVectorElementType::INT8: { + std::vector values(num_elements); + std::memcpy(values.data(), bytes, num_elements); + RETURN_IF_ERROR(set_nearest(values.data(), LANCE_DTYPE_INT8)); + break; + } + default: + return Status::NotSupported("unsupported Lance query vector element type: {}", + static_cast(query.element_type)); + } + + { + // FE plans an omitted metric as L2; never let indexed splits choose another default. + const auto requested_metric = + !vector.__isset.metric || vector.metric == TVectorMetric::DEFAULT + ? TVectorMetric::L2 + : vector.metric; + LanceMetricType metric; + switch (requested_metric) { + case TVectorMetric::L2: + metric = LANCE_METRIC_L2; + break; + case TVectorMetric::COSINE: + metric = LANCE_METRIC_COSINE; + break; + case TVectorMetric::DOT_PRODUCT: + metric = LANCE_METRIC_DOT; + break; + case TVectorMetric::HAMMING: + metric = LANCE_METRIC_HAMMING; + break; + default: + return Status::NotSupported("unsupported Lance vector metric: {}", + static_cast(vector.metric)); + } + if (lance_scanner_set_metric(scanner, metric) != 0) { + return lance_error("set Lance vector metric"); + } + } + + if (vector.__isset.distance_lower_bound || vector.__isset.distance_upper_bound) { + const float lower = static_cast(vector.distance_lower_bound); + const float upper = static_cast(vector.distance_upper_bound); + // nearest() clears the range; install bounds only after setting the query vector. + if (lance_scanner_set_distance_range( + scanner, vector.__isset.distance_lower_bound ? &lower : nullptr, + vector.__isset.distance_upper_bound ? &upper : nullptr) != 0) { + return lance_error("set Lance vector distance range"); + } + } + + // Exact refinement validates stored elements and gives indexed and unindexed candidates + // the same row-level score before either path truncates its results. + if (query.__isset.num_vectors && lance_scanner_set_refine_factor(scanner, 1) != 0) { + return lance_error("enable Lance multi-vector refinement"); + } + if (request.__isset.vector_search_options) { + const auto& options = request.vector_search_options; + if (options.__isset.query_parallelism && + lance_scanner_set_query_parallelism(scanner, options.query_parallelism) != 0) { + return lance_error("set Lance vector query parallelism"); + } + if (options.__isset.nprobes && + lance_scanner_set_nprobes(scanner, static_cast(options.nprobes)) != 0) { + return lance_error("set Lance vector nprobes"); + } + if (options.__isset.refine_factor && + lance_scanner_set_refine_factor(scanner, + static_cast(options.refine_factor)) != 0) { + return lance_error("set Lance vector refine factor"); + } + if (options.__isset.ef && + lance_scanner_set_ef(scanner, static_cast(options.ef)) != 0) { + return lance_error("set Lance vector ef"); + } + } + // FE plans a split without index segments as a flat search, and EXPLAIN and the profile count + // it as one. Lance would otherwise search it with the first index on the column in manifest + // order, which need not be the index FE selected. A split with segments searches exactly + // those; one that also carries use_index=false was rejected above. + if (lance_scanner_set_use_index(scanner, segment_count > 0) != 0) { + return lance_error("set Lance vector use_index"); + } + if (lance_scanner_set_offset(scanner, vector.offset) != 0) { + return lance_error("set Lance vector offset"); + } + if (lance_scanner_set_limit(scanner, vector.top_k) != 0) { + return lance_error("set Lance vector result limit"); + } + const auto fragment_count = static_cast(fragment_ids.size()); + if (segment_count > 0) { + COUNTER_UPDATE(_planned_index_segment_count, static_cast(segment_count)); + COUNTER_UPDATE(_planned_indexed_fragment_count, fragment_count); + } else { + COUNTER_UPDATE(_planned_flat_search_fragment_count, fragment_count); + } + return Status::OK(); +} + +Status LanceTableReader::_configure_full_text_search(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const { + DORIS_CHECK(scanner != nullptr); + DORIS_CHECK(_fts_query_context != nullptr); + DORIS_CHECK(_scan_params != nullptr); + // FTS fragment IDs describe the selected segment's coverage for planning and profiling. They + // are not installed as a generic fragment filter because lance-c rejects combining one with a + // prepared FTS context; the segment UUID is the execution boundary. + std::vector fragment_ids; + RETURN_IF_ERROR(parse_fragment_ids(lance_params, &fragment_ids)); + std::vector segment_uuids; + size_t segment_count = 0; + RETURN_IF_ERROR(parse_index_segment_uuids(lance_params, &segment_uuids, &segment_count)); + if (segment_count == 0) { + return Status::InvalidArgument( + "Lance full-text search split requires at least one FTS index segment UUID"); + } + const auto& full_text = + _scan_params->lance_scan_params.external_search_request.search_query.full_text_search; + const auto& request = _scan_params->lance_scan_params.external_search_request; + const bool prefilter = request.__isset.prefilter && request.prefilter; + if (lance_scanner_set_prefilter(scanner, prefilter) != 0) { + return lance_error("set Lance full-text search prefilter"); + } + if (lance_scanner_set_fts_query_context(scanner, _fts_query_context) != 0) { + return lance_error("attach Lance FTS query context"); + } + if (lance_scanner_set_fts_index_segments(scanner, segment_uuids.data(), segment_count) != 0) { + return lance_error("set Lance FTS scanner index segments"); + } + if (lance_scanner_set_limit(scanner, full_text.top_k) != 0) { + return lance_error("set Lance FTS scanner candidate limit"); + } + COUNTER_UPDATE(_planned_index_segment_count, static_cast(segment_count)); + COUNTER_UPDATE(_planned_indexed_fragment_count, static_cast(fragment_ids.size())); + return Status::OK(); +} + +void LanceTableReader::_collect_scan_statistics(void* callback_ctx, const void* opaque_statistics) { + const auto* statistics = static_cast(opaque_statistics); + if (callback_ctx == nullptr || statistics == nullptr) { + LOG(WARNING) << "Lance scan statistics callback received a null argument"; + return; + } + + auto* reader = static_cast(callback_ctx); + const auto update_counter = [](RuntimeProfile::Counter* counter, uint64_t value, + std::string_view metric_name) { + if (counter == nullptr) { + return; + } + if (value > static_cast(std::numeric_limits::max())) { + LOG(WARNING) << "Ignoring Lance scan metric '" << metric_name << "' with value " + << value << " because it exceeds INT64_MAX"; + return; + } + COUNTER_UPDATE(counter, static_cast(value)); + }; + + update_counter(reader->_execution_iops, statistics->iops, "iops"); + update_counter(reader->_execution_requests, statistics->requests, "requests"); + update_counter(reader->_execution_bytes_read, statistics->bytes_read, "bytes_read"); + update_counter(reader->_index_object_loads, statistics->indices_loaded, "indices_loaded"); + update_counter(reader->_index_components_loaded, statistics->index_partitions_loaded, + "index_partitions_loaded"); + update_counter(reader->_index_comparisons, statistics->index_comparisons, "index_comparisons"); + + if (statistics->metrics_len != 0 && statistics->metrics == nullptr) { + LOG(WARNING) << "Ignoring malformed Lance scan statistics: metrics is NULL while " + << "metrics_len is " << statistics->metrics_len; + return; + } + for (size_t index = 0; index < statistics->metrics_len; ++index) { + const auto& metric = statistics->metrics[index]; + if (metric.name_len != 0 && metric.name == nullptr) { + LOG(WARNING) << "Ignoring malformed Lance scan metric at index " << index + << ": name is NULL while name_len is " << metric.name_len; + continue; + } + const std::string_view name(metric.name == nullptr ? "" : metric.name, metric.name_len); + size_t metric_index = 0; + for (const auto& definition : LANCE_SCAN_METRICS) { + if (definition.native_name != name) { + ++metric_index; + continue; + } + const auto expected_kind = definition.unit == TUnit::TIME_NS + ? LANCE_SCAN_METRIC_TIME_NANOSECONDS + : LANCE_SCAN_METRIC_COUNT; + // _init_scanner_profile() registers one counter per LANCE_SCAN_METRICS entry before + // the callback is installed. + DCHECK_LT(metric_index, reader->_scan_metric_counters.size()); + if (metric.kind == expected_kind) { + update_counter(reader->_scan_metric_counters[metric_index], metric.value, name); + } + break; + } + } +} + +void LanceTableReader::_close_scanner() { + if (_scanner != nullptr) { + lance_scanner_close(_scanner); + _scanner = nullptr; + } + _scanner_batch_size = 0; +} + +void LanceTableReader::_close_dataset() { + if (_fts_query_context != nullptr) { + lance_fts_query_context_close(_fts_query_context); + _fts_query_context = nullptr; + } + if (_dataset != nullptr) { + _collect_data_cache_statistics(); + lance_dataset_close(_dataset); + _dataset = nullptr; + } + _dataset_schema.reset(); + _record_batch_converter.reset_schema(); +} + +void LanceTableReader::_collect_data_cache_statistics() { + if (_dataset == nullptr) { + return; + } + + LanceDataCacheStatistics statistics {}; + if (lance_dataset_get_data_cache_statistics(_dataset, &statistics) != 0) { + const auto status = lance_error("get Lance data cache statistics"); + LOG(WARNING) << "Failed to collect Lance data cache statistics: " << status.to_string(); + return; + } + + const auto set_counter = [](RuntimeProfile::Counter* counter, uint64_t value, + std::string_view metric_name) { + if (counter == nullptr) { + return; + } + if (value > static_cast(std::numeric_limits::max())) { + LOG(WARNING) << "Ignoring Lance data cache metric '" << metric_name << "' with value " + << value << " because it exceeds INT64_MAX"; + return; + } + COUNTER_SET(counter, static_cast(value)); + }; + set_counter(_data_cache_bytes_read_from_cache, statistics.bytes_read_from_cache, + "bytes_read_from_cache"); + set_counter(_data_cache_bytes_read_from_remote, statistics.bytes_read_from_remote, + "bytes_read_from_remote"); +} + +Status LanceTableReader::_fill_block_from_lance_batch(LanceBatch* batch, Block* block, + size_t* rows) { + DORIS_CHECK(batch != nullptr); + DORIS_CHECK(block != nullptr); + DORIS_CHECK(rows != nullptr); + ArrowArray array {}; + ArrowSchema schema {}; + if (lance_batch_to_arrow(batch, &array, &schema) != 0) { + return lance_error("export Lance batch to Arrow"); + } + auto result = arrow::ImportRecordBatch(&array, &schema); + if (!result.ok()) { + if (array.release != nullptr) { + array.release(&array); + } + if (schema.release != nullptr) { + schema.release(&schema); + } + return Status::InternalError("import Lance Arrow batch failed: {}", + result.status().message()); + } + + return _record_batch_converter.convert_record_batch_to_block( + std::move(result).ValueUnsafe(), block, _global_rowid_context, rows); +} + +Status LanceTableReader::_dataset_key(const TFileRangeDesc& range, DatasetKey* key) const { + const auto& params = range.table_format_params.lance_params; + key->uri = params.dataset_uri; + key->version = params.version; + return build_lance_storage_options(_scan_params, &key->storage_options); +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h new file mode 100644 index 00000000000000..b0284e768df353 --- /dev/null +++ b/be/src/format_v2/table/lance_reader.h @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/lance/lance_record_batch_converter.h" +#include "format_v2/table_reader.h" +#include "runtime/runtime_profile.h" + +struct LanceBatch; +struct LanceDataset; +struct LanceFtsQueryContext; +struct LanceScanner; + +namespace doris { +class ShardedKVCache; +} + +namespace doris::format::lance { + +// A FORMAT_LANCE table reader. Unlike file formats such as Parquet, a Lance split is not a +// physical-file range. It either selects fragments from a fixed snapshot or scans the whole +// latest snapshot, so the dataset is owned by this table reader and each split owns its scanner. +class LanceTableReader final : public TableReader { +public: + ~LanceTableReader() override; + + // Fetch the schema of a Lance dataset without initializing the scan path. Version zero opens + // the latest snapshot, which is used by backend-local TVF schema discovery. + Status fetch_schema(const TFileRangeDesc& range, const TFileScanRangeParams& scan_params, + std::vector* column_names, + std::vector* column_types) const; + + Status init(TableReadOptions&& options) override; + Status prepare_split(const SplitReadOptions& options) override; + Status get_block(Block* block, bool* eos) override; + // Fetch top-level projected columns by native Lance row IDs from one fixed dataset snapshot. + // Input order and duplicates are preserved by lance-c. Missing rows are rejected because row + // IDs produced by phase one must still exist in the same snapshot during materialization. + Status read_by_row_ids(const TFileRangeDesc& range, const std::vector& row_ids, + Block* block); + Status abort_split() override; + Status close() override; + +private: + struct DatasetKey { + std::string uri; + int64_t version = 0; + std::vector storage_options; + bool operator==(const DatasetKey&) const = default; + }; + + Status _resolve_search_kind(); + Status _validate_external_search_request() const; + Status _ensure_dataset_open(const TFileRangeDesc& range, bool prepare_fts_context = true); + Status _open_dataset(const DatasetKey& key); + Status _prepare_fts_query_context(); + Status _open_scanner(const TFileRangeDesc& range); + void _init_scanner_profile(); + Status _configure_scan_options(LanceScanner* scanner) const; + Status _configure_normal_scan(LanceScanner* scanner, const TLanceFileDesc& lance_params) const; + Status _configure_vector_search(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const; + Status _configure_full_text_search(LanceScanner* scanner, + const TLanceFileDesc& lance_params) const; + // Collect the cumulative statistics owned by this dataset handle. The Lance-C API returns an + // absolute snapshot, so this method replaces rather than increments the profile counters. + void _collect_data_cache_statistics(); + // Keep lance-c's anonymous statistics typedef out of this header. _open_scanner installs the + // strongly typed C callback adapter before forwarding the borrowed value here. + static void _collect_scan_statistics(void* callback_ctx, const void* opaque_statistics); + void _close_scanner(); + void _close_dataset(); + Status _fill_block_from_lance_batch(LanceBatch* batch, Block* block, size_t* rows); + Status _dataset_key(const TFileRangeDesc& range, DatasetKey* key) const; + + LanceDataset* _dataset = nullptr; + std::shared_ptr _dataset_schema; + LanceScanner* _scanner = nullptr; + ShardedKVCache* _runtime_filter_cache = nullptr; + std::optional _opened_dataset_key; + LanceRecordBatchConverter _record_batch_converter; + size_t _scanner_batch_size = 0; + RuntimeProfile::Counter* _planned_index_segment_count = nullptr; + RuntimeProfile::Counter* _planned_indexed_fragment_count = nullptr; + RuntimeProfile::Counter* _planned_flat_search_fragment_count = nullptr; + RuntimeProfile::Counter* _dataset_open_time = nullptr; + RuntimeProfile::Counter* _scanner_configure_time = nullptr; + RuntimeProfile::Counter* _runtime_filter_sql_translation_time = nullptr; + RuntimeProfile::Counter* _scanner_next_time = nullptr; + RuntimeProfile::Counter* _arrow_to_doris_block_time = nullptr; + RuntimeProfile::Counter* _row_id_take_read_time = nullptr; + RuntimeProfile::Counter* _row_id_fetch_total_time = nullptr; + RuntimeProfile::Counter* _execution_iops = nullptr; + RuntimeProfile::Counter* _execution_requests = nullptr; + RuntimeProfile::Counter* _execution_bytes_read = nullptr; + // Pre-registered before installing Lance's callback so it only updates existing counters. + std::vector _scan_metric_counters; + RuntimeProfile::Counter* _data_cache_bytes_read_from_cache = nullptr; + RuntimeProfile::Counter* _data_cache_bytes_read_from_remote = nullptr; + RuntimeProfile::Counter* _index_object_loads = nullptr; + RuntimeProfile::Counter* _index_components_loaded = nullptr; + RuntimeProfile::Counter* _index_comparisons = nullptr; + LanceFtsQueryContext* _fts_query_context = nullptr; + SearchKind _search_kind = SearchKind::NORMAL; + bool _eof = false; +}; + +} // namespace doris::format::lance diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 5a8d5dea5e7765..fba778818a7724 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -111,6 +111,8 @@ std::string file_format_to_string(FileFormat format) { return "ARROW"; case FileFormat::WAL: return "WAL"; + case FileFormat::LANCE: + return "LANCE"; } return "UNKNOWN"; } diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 154ba5e64310d4..264ab5524eb0b0 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -230,6 +230,7 @@ class TableReader { _current_file_description->is_immutable); return _current_task->data_file->is_immutable; } + bool TEST_conjuncts_empty() const { return _conjuncts.empty(); } #endif // Prepare for reading a new split/task. diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index d64349a6d297f6..e7e8ebade3fbb6 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -77,6 +77,7 @@ #include "format/text/text_reader.h" #include "format_v2/column_data.h" #include "format_v2/schema_reader.h" +#include "format_v2/table/lance_reader.h" #include "io/fs/local_file_system.h" #include "io/fs/stream_load_pipe.h" #include "io/io_common.h" @@ -927,6 +928,39 @@ void PInternalService::fetch_table_schema(google::protobuf::RpcController* contr fetch_schema_batch_size, io_ctx.get(), io_ctx); break; } + case TFileFormatType::FORMAT_LANCE: { + std::vector lance_col_names; + std::vector lance_col_types; + auto lance_reader = std::make_unique(); + st = lance_reader->fetch_schema(range, params, &lance_col_names, &lance_col_types); + if (!st.ok()) { + LOG(WARNING) << "fetch Lance table schema failed, errmsg=" << st; + st.to_protobuf(result->mutable_status()); + return; + } + DORIS_CHECK_EQ(lance_col_names.size(), lance_col_types.size()); + result->set_column_nums(lance_col_names.size()); + for (const auto& col_name : lance_col_names) { + result->add_column_names(col_name); + } + for (const auto& col_type : lance_col_types) { + DORIS_CHECK(col_type != nullptr); + PTypeDesc* type_desc = result->add_column_types(); + if (col_type->is_null_literal()) { + PTypeNode* node = type_desc->add_types(); + node->set_type(TTypeNodeType::SCALAR); + node->mutable_scalar_type()->set_type(TPrimitiveType::NULL_TYPE); + } else if (col_type->get_primitive_type() == INVALID_TYPE) { + PTypeNode* node = type_desc->add_types(); + node->set_type(TTypeNodeType::SCALAR); + node->mutable_scalar_type()->set_type(TPrimitiveType::UNSUPPORTED); + } else { + col_type->to_protobuf(type_desc); + } + } + st.to_protobuf(result->mutable_status()); + return; + } default: st = Status::InternalError("Not supported file format in fetch table schema: {}", params.format_type); diff --git a/be/src/storage/utils.h b/be/src/storage/utils.h index fdeea6cad10c2d..73c171457c33cb 100644 --- a/be/src/storage/utils.h +++ b/be/src/storage/utils.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -228,15 +229,89 @@ struct RowLocation { using RowLocationSet = std::set; using RowLocationPairList = std::list>; +// Wire-protocol values: never reorder or reuse an existing value. A new value may use a new +// encoded structure and size, provided its decoder keeps supporting all older values. +enum class ROW_VERSION : uint8_t { + // The row ID is a uint32 ordinal local to the FileMapping identified by file_id. + FILE_LOCAL_ROW_ID = 0, + // The row ID is an opaque uint64 ID in a fixed Lance dataset snapshot. + LANCE_DATASET_ROW_ID = 1, +}; + +/* + * A serialized global row location has a fixed size of 24 bytes. The version determines how the + * bytes at offsets 4..7 and 16..23 must be interpreted: + * + * FILE_LOCAL_ROW_ID (version = 0), used by Doris, Parquet, and ORC: + * + * byte offset 0 1..7 8..15 16..19 20..23 + * +--------+----------------+---------------+-----------+-----------+ + * | ver=0 | reserved | backend_id | file_id | row_id | + * +--------+----------------+---------------+-----------+-----------+ + * uint8 7 bytes int64 uint32 uint32 + * + * row_id is an ordinal local to the FileMapping selected by file_id. + * + * LANCE_DATASET_ROW_ID (version = 1), used by Lance: + * + * byte offset 0 1..3 4..7 8..15 16..23 + * +--------+----------+-------------+---------------+----------------+ + * | ver=1 | reserved | file_id | backend_id | lance_row_id | + * +--------+----------+-------------+---------------+----------------+ + * uint8 3 bytes uint32 int64 uint64 + * + * lance_row_id is an opaque row ID in the fixed dataset snapshot recorded by the FileMapping. + * + * The first union reuses four bytes that are padding in version 0 as Lance's file_id in version 1. + * The second union reuses the original {uint32 file_id, uint32 row_id} payload as one uint64 Lance + * row ID. Therefore, always check version before reading either union. + */ struct GlobalRowLoacationV2 { + static constexpr uint8_t VERSION = static_cast(ROW_VERSION::FILE_LOCAL_ROW_ID); + + struct FileLocalRowId { + uint32_t file_id; + uint32_t row_id; + }; + GlobalRowLoacationV2(uint8_t ver, uint64_t bid, uint32_t fid, uint32_t rid) - : version(ver), backend_id(bid), file_id(fid), row_id(rid) {} + : version(ver), + reserved_for_file_local(0), + backend_id(bid), + file_local {.file_id = fid, .row_id = rid} {} + GlobalRowLoacationV2(ROW_VERSION ver, uint64_t bid, uint32_t fid, uint64_t rid) + : version(static_cast(ver)), + lance_file_id(fid), + backend_id(bid), + lance_row_id(rid) {} + uint8_t version; + uint8_t reserved_before_file_id[3] {}; + union { + // version 0: offsets 4..7 remain reserved, preserving the original V2 layout. + uint32_t reserved_for_file_local; + // version 1: offsets 4..7 identify the FileMapping for lance_row_id. + uint32_t lance_file_id; + }; int64_t backend_id; - uint32_t file_id; - uint32_t row_id; - - auto operator<=>(const GlobalRowLoacationV2&) const = default; + union { + // version 0: file_id is at offset 16 and its uint32 row ordinal is at offset 20. + FileLocalRowId file_local; + // version 1: offsets 16..23 are one opaque uint64 Lance row ID. + uint64_t lance_row_id; + }; }; +static_assert(sizeof(GlobalRowLoacationV2) == 24); +static_assert(sizeof(GlobalRowLoacationV2::FileLocalRowId) == 8); +static_assert(offsetof(GlobalRowLoacationV2, version) == 0); +static_assert(offsetof(GlobalRowLoacationV2, reserved_before_file_id) == 1); +static_assert(offsetof(GlobalRowLoacationV2, reserved_for_file_local) == 4); +static_assert(offsetof(GlobalRowLoacationV2, lance_file_id) == 4); +static_assert(offsetof(GlobalRowLoacationV2, backend_id) == 8); +static_assert(offsetof(GlobalRowLoacationV2, file_local) == 16); +static_assert(offsetof(GlobalRowLoacationV2::FileLocalRowId, file_id) == 0); +static_assert(offsetof(GlobalRowLoacationV2::FileLocalRowId, row_id) == 4); +static_assert(offsetof(GlobalRowLoacationV2, lance_row_id) == 16); + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp b/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp index 51dec6fc83ba4e..4b08e2c23b7bff 100644 --- a/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp @@ -388,6 +388,64 @@ TEST_F(DataTypeDateTimeV2SerDeTest, RejectsInvalidDate64Values) { expect_invalid(out_of_range, "outside the Doris DATE range"); } +TEST_F(DataTypeDateTimeV2SerDeTest, RejectsInvalidDate32Values) { + const auto make_date32 = [](int32_t days) { + arrow::Date32Builder builder; + EXPECT_TRUE(builder.Append(days).ok()); + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; + }; + cctz::time_zone tz; + + // 1970-01-01 + 2932896 days is 9999-12-31, the last Doris DATE. + auto last_date = ColumnDateV2::create(); + auto status = + serde_date_v2->read_column_from_arrow(*last_date, make_date32(2932896).get(), 0, 1, tz); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(1, last_date->size()); + EXPECT_EQ(9999, last_date->get_data()[0].year()); + EXPECT_EQ(12, last_date->get_data()[0].month()); + EXPECT_EQ(31, last_date->get_data()[0].day()); + + // 10000-01-01, -719529 (the day before 0000-01-01 in Arrow's proleptic Gregorian calendar) + // and both int32 extremes used to read as a non-null 0000-00-00; INT32_MAX also overflowed + // the 32-bit day number. + for (const int32_t days : {2932897, -719529, std::numeric_limits::min(), + std::numeric_limits::max()}) { + SCOPED_TRACE(days); + auto column = ColumnDateV2::create(); + status = serde_date_v2->read_column_from_arrow(*column, make_date32(days).get(), 0, 1, tz); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, + status.to_string().find("Arrow Date32 value is outside the Doris DATE range")) + << status.to_string(); + } + + // A NULL slot keeps whatever payload its writer left; it must not be decoded. Validity + // bitmap 0b10 marks row 0 NULL over the out-of-range payload and row 1 valid. + arrow::Date32Builder builder; + ASSERT_TRUE(builder.AppendValues({2932897, 0}).ok()); + std::shared_ptr payload_array; + ASSERT_TRUE(builder.Finish(&payload_array).ok()); + auto data = payload_array->data()->Copy(); + data->buffers[0] = arrow::Buffer::FromVector(std::vector {0x02}); + data->SetNullCount(1); + const auto nullable_array = arrow::MakeArray(data); + DataTypeNullableSerDe nullable_serde(serde_date_v2); + auto nullable_column = ColumnNullable::create(ColumnDateV2::create(), ColumnUInt8::create()); + status = + nullable_serde.read_column_from_arrow(*nullable_column, nullable_array.get(), 0, 2, tz); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(2, nullable_column->size()); + EXPECT_TRUE(nullable_column->is_null_at(0)); + EXPECT_FALSE(nullable_column->is_null_at(1)); + const auto& dates = assert_cast(nullable_column->get_nested_column()); + EXPECT_EQ(1970, dates.get_data()[1].year()); + EXPECT_EQ(1, dates.get_data()[1].month()); + EXPECT_EQ(1, dates.get_data()[1].day()); +} + TEST_F(DataTypeDateTimeV2SerDeTest, ArrowTimeToTimeV2) { const auto read_time = [](arrow::ArrayBuilder* builder, int scale, const std::vector& expected) { diff --git a/be/test/exec/operator/materialization_shared_state_test.cpp b/be/test/exec/operator/materialization_shared_state_test.cpp index fb3c31493a6471..4859612d62990a 100644 --- a/be/test/exec/operator/materialization_shared_state_test.cpp +++ b/be/test/exec/operator/materialization_shared_state_test.cpp @@ -17,6 +17,9 @@ #include +#include +#include + #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" @@ -29,11 +32,30 @@ namespace doris { namespace { -void add_request_row(PRequestBlockDesc* request_block_desc, uint32_t row_id, uint32_t file_id) { +void add_request_row(PRequestBlockDesc* request_block_desc, uint64_t row_id, uint32_t file_id) { request_block_desc->add_row_id(row_id); request_block_desc->add_file_id(file_id); } +void set_lance_fetch_profile(PMultiGetBlockV2* response_block, int64_t scale) { + RuntimeProfile profile("ExternalRowIDFetcher"); + profile.add_info_string("LanceDatasetOpenTime", std::to_string(scale) + "ns"); + profile.add_info_string("LanceRowIdTakeReadTime", std::to_string(2 * scale) + "ns"); + profile.add_info_string("LanceArrowToDorisBlockTime", std::to_string(3 * scale) + "ns"); + profile.add_info_string("LanceRowIdFetchTotalTime", std::to_string(4 * scale) + "ns"); + profile.add_info_string("LanceRowIdFetchRows", std::to_string(5 * scale)); + profile.add_info_string("LanceRowIdFetchCalls", "1"); + profile.add_info_string("LanceDataCacheBytesReadFromCache", "64.00 KB"); + profile.add_info_string("LanceDataCacheBytesReadFromRemote", "32.00 KB"); + profile.add_info_string("ScannersRunningTime", "0ms"); + profile.add_info_string("InitReaderAvgTime", "0ms"); + profile.add_info_string("GetBlockAvgTime", "0ms"); + profile.add_info_string("FileReadLines", "[]"); + profile.add_info_string("FileReadBytes", "[]"); + profile.add_info_string("FileReadTime", "[]"); + profile.to_proto(response_block->mutable_profile(), 2); +} + } // namespace class MaterializationSharedStateTest : public testing::Test { @@ -125,6 +147,73 @@ TEST_F(MaterializationSharedStateTest, TestCreateMultiGetResult) { // Verify block_order_results EXPECT_EQ(_shared_state->block_order_results.size(), columns.size()); EXPECT_EQ(_shared_state->eos, true); + const auto& backend1_request = + _shared_state->rpc_struct_map[_backend_id1].request.request_block_descs(0); + ASSERT_EQ(backend1_request.row_id_size(), 1); + EXPECT_EQ(backend1_request.row_id(0), 1); + const auto& backend2_request = + _shared_state->rpc_struct_map[_backend_id2].request.request_block_descs(0); + ASSERT_EQ(backend2_request.row_id_size(), 1); + EXPECT_EQ(backend2_request.row_id(0), 2); +} + +TEST_F(MaterializationSharedStateTest, TestCreateMultiGetResultWithUint64RowId) { + Columns columns; + auto rowid_col = _string_type->create_column(); + auto* col_data = reinterpret_cast(rowid_col.get()); + + constexpr uint64_t large_row_id = + static_cast(std::numeric_limits::max()) + 17; + GlobalRowLoacationV2 location(ROW_VERSION::LANCE_DATASET_ROW_ID, _backend_id1, 7, large_row_id); + col_data->insert_data(reinterpret_cast(&location), sizeof(location)); + columns.push_back(std::move(rowid_col)); + + ASSERT_TRUE(_shared_state->create_muiltget_result(columns, false).ok()); + + const auto& request = _shared_state->rpc_struct_map[_backend_id1].request; + ASSERT_EQ(request.request_block_descs_size(), 1); + const auto& request_block = request.request_block_descs(0); + ASSERT_EQ(request_block.row_id_size(), 1); + EXPECT_EQ(request_block.row_id(0), large_row_id); + ASSERT_EQ(request_block.file_id_size(), 1); + EXPECT_EQ(request_block.file_id(0), 7); + EXPECT_EQ(request_block.row_location_version(), + static_cast(ROW_VERSION::LANCE_DATASET_ROW_ID)); + EXPECT_EQ(_shared_state->block_order_results[0][0], _backend_id1); +} + +TEST_F(MaterializationSharedStateTest, TestRejectUnknownRowLocationVersionWithFutureSize) { + Columns columns; + auto rowid_col = _string_type->create_column(); + auto* col_data = reinterpret_cast(rowid_col.get()); + + std::array future_location {}; + future_location[0] = 99; + col_data->insert_data(future_location.data(), future_location.size()); + columns.push_back(std::move(rowid_col)); + + const Status st = _shared_state->create_muiltget_result(columns, false); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("unsupported global row location version: 99"), + std::string::npos); + EXPECT_NE(st.to_string().find("encoded_size=32"), std::string::npos); +} + +TEST_F(MaterializationSharedStateTest, TestRejectKnownRowLocationVersionWithInvalidSize) { + Columns columns; + auto rowid_col = _string_type->create_column(); + auto* col_data = reinterpret_cast(rowid_col.get()); + + std::array invalid_location {}; + invalid_location[0] = static_cast(ROW_VERSION::LANCE_DATASET_ROW_ID); + col_data->insert_data(invalid_location.data(), invalid_location.size()); + columns.push_back(std::move(rowid_col)); + + const Status st = _shared_state->create_muiltget_result(columns, false); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("invalid global row location size for version 1"), + std::string::npos); + EXPECT_NE(st.to_string().find("actual=25, expected=24"), std::string::npos); } TEST_F(MaterializationSharedStateTest, TestMergeMultiResponse) { @@ -172,6 +261,7 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponse) { auto s = resp_block1.serialize(0, serialized_block, &uncompressed_size, &compressed_size, &compress_time, CompressionTypePB::LZ4); EXPECT_TRUE(s.ok()); + set_lance_fetch_profile(response_.mutable_blocks(0), 10); _shared_state->rpc_struct_map[_backend_id1].response = std::move(response_); // init the response blocks @@ -198,6 +288,7 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponse) { auto s = resp_block2.serialize(0, serialized_block, &uncompressed_size, &compressed_size, &compress_time, CompressionTypePB::LZ4); EXPECT_TRUE(s.ok()); + set_lance_fetch_profile(response_.mutable_blocks(0), 1); _shared_state->rpc_struct_map[_backend_id2].response = std::move(response_); } @@ -224,6 +315,20 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponse) { EXPECT_EQ(merged_value_col->get_data_at(1).data, nullptr); // Second value from BE1, replace by null EXPECT_EQ(*((int*)merged_value_col->get_data_at(2).data), 200); // Third value from BE2 + const auto& backend1_info = _shared_state->backend_profile_info_string.at(_backend_id1); + EXPECT_EQ("10ns, ", fmt::to_string(backend1_info.at("LanceDatasetOpenTime"))); + EXPECT_EQ("20ns, ", fmt::to_string(backend1_info.at("LanceRowIdTakeReadTime"))); + EXPECT_EQ("30ns, ", fmt::to_string(backend1_info.at("LanceArrowToDorisBlockTime"))); + EXPECT_EQ("40ns, ", fmt::to_string(backend1_info.at("LanceRowIdFetchTotalTime"))); + EXPECT_EQ("50, ", fmt::to_string(backend1_info.at("LanceRowIdFetchRows"))); + EXPECT_EQ("1, ", fmt::to_string(backend1_info.at("LanceRowIdFetchCalls"))); + EXPECT_EQ("64.00 KB, ", fmt::to_string(backend1_info.at("LanceDataCacheBytesReadFromCache"))); + EXPECT_EQ("32.00 KB, ", fmt::to_string(backend1_info.at("LanceDataCacheBytesReadFromRemote"))); + const auto& backend2_info = _shared_state->backend_profile_info_string.at(_backend_id2); + EXPECT_EQ("1ns, ", fmt::to_string(backend2_info.at("LanceDatasetOpenTime"))); + EXPECT_EQ("2ns, ", fmt::to_string(backend2_info.at("LanceRowIdTakeReadTime"))); + EXPECT_EQ("3ns, ", fmt::to_string(backend2_info.at("LanceArrowToDorisBlockTime"))); + EXPECT_EQ("4ns, ", fmt::to_string(backend2_info.at("LanceRowIdFetchTotalTime"))); } TEST_F(MaterializationSharedStateTest, TestMergeMultiResponseMultiBlocks) { diff --git a/be/test/exec/rowid_fetcher_test.cpp b/be/test/exec/rowid_fetcher_test.cpp index b0478cfc4abd74..c90dbf37d196a1 100644 --- a/be/test/exec/rowid_fetcher_test.cpp +++ b/be/test/exec/rowid_fetcher_test.cpp @@ -19,6 +19,8 @@ #include +#include +#include #include #include @@ -146,6 +148,8 @@ TEST_F(RowIdStorageReaderTest, ExternalScannerSelectionKeepsUnsupportedFormatsOn TFileRangeDesc range; range.__set_format_type(TFileFormatType::FORMAT_JNI); EXPECT_FALSE(RowIdStorageReader::should_use_file_scanner_v2(options, params, range)); + range.__set_format_type(TFileFormatType::FORMAT_LANCE); + EXPECT_FALSE(RowIdStorageReader::should_use_file_scanner_v2(options, params, range)); range.__set_format_type(TFileFormatType::FORMAT_ORC); TTableFormatFileDesc table; table.__set_table_format_type("transactional_hive"); @@ -501,4 +505,84 @@ TEST_F(RowIdStorageReaderTest, RpcHelperReportsThrownExceptionOnThreadPoolPath) EXPECT_NE(response.status().error_msgs(0).find("invalid TTypeNodeType"), std::string::npos); } +// Phase-two row fetch decodes a PRequestBlockDesc built by another BE. Each case below is a request +// that a newer sender (an unknown ROW_VERSION) or a malformed one could send; it must fail before +// any per-row lookup instead of being read as a FILE_LOCAL_ROW_ID request. +class RowIdFetchRequestValidationTest : public RowIdStorageReaderTest { +protected: + void SetUp() override { + _saved_id_manager = ExecEnv::GetInstance()->get_id_manager(); + ExecEnv::GetInstance()->set_id_manager(&_id_manager); + _request.mutable_query_id()->set_hi(37); + _request.mutable_query_id()->set_lo(1); + _id_file_map = _id_manager.add_id_file_map(_request.query_id(), 1024); + _internal_file_id = _id_file_map->get_file_mapping_id( + std::make_shared(/*tablet_id=*/1, RowsetId(), /*segment_id=*/0)); + _block_desc = _request.add_request_block_descs(); + make_slot({}).to_protobuf(_block_desc->add_slots()); + } + + void TearDown() override { ExecEnv::GetInstance()->set_id_manager(_saved_id_manager); } + + Status fetch() const { + PMultiGetResponseV2 response; + return RowIdStorageReader::read_by_rowids(_request, &response); + } + + IdManager _id_manager; + IdManager* _saved_id_manager = nullptr; + PMultiGetRequestV2 _request; + std::shared_ptr _id_file_map; + PRequestBlockDesc* _block_desc = nullptr; + uint32_t _internal_file_id = 0; +}; + +TEST_F(RowIdFetchRequestValidationTest, RejectsMismatchedFileIdAndRowIdCounts) { + _block_desc->add_file_id(_internal_file_id); + _block_desc->add_row_id(0); + _block_desc->add_row_id(1); + + const Status st = fetch(); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("file_ids=1, row_ids=2"), std::string::npos) << st; +} + +TEST_F(RowIdFetchRequestValidationTest, RejectsUnknownRowLocationVersion) { + _block_desc->add_file_id(_internal_file_id); + _block_desc->add_row_id(0); + _block_desc->set_row_location_version(99); + + const Status st = fetch(); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("unsupported row location version in fetch: 99"), + std::string::npos) + << st; +} + +TEST_F(RowIdFetchRequestValidationTest, RejectsLanceRowIdForInternalMapping) { + _block_desc->add_file_id(_internal_file_id); + _block_desc->add_row_id(0); + _block_desc->set_row_location_version(static_cast(ROW_VERSION::LANCE_DATASET_ROW_ID)); + + const Status st = fetch(); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("Lance dataset row IDs require an external file mapping"), + std::string::npos) + << st; +} + +TEST_F(RowIdFetchRequestValidationTest, RejectsInternalRowIdBeyondUint32) { + // 2^32 is representable on the uint64 wire but is not a segment ordinal. Truncating it would + // fetch row 0 of the segment. + constexpr uint64_t row_id = static_cast(std::numeric_limits::max()) + 1; + _block_desc->add_file_id(_internal_file_id); + _block_desc->add_row_id(row_id); + + const Status st = fetch(); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("internal row id exceeds uint32 range: 4294967296"), + std::string::npos) + << st; +} + } // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index b38caa0a7bcebe..86abaf86958c80 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -477,9 +477,10 @@ TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesAndLoads) EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); params.__set_format_type(TFileFormatType::FORMAT_JNI); EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); + params.__set_format_type(TFileFormatType::FORMAT_LANCE); + EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); - const std::vector unsupported_formats {TFileFormatType::FORMAT_ES_HTTP, - TFileFormatType::FORMAT_LANCE}; + const std::vector unsupported_formats {TFileFormatType::FORMAT_ES_HTTP}; for (const auto format : unsupported_formats) { params.__set_format_type(format); EXPECT_FALSE( @@ -525,6 +526,22 @@ TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesAndLoads) EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(query_options, false, params)); } +// enable_file_scanner_v2 is fuzzy=true, and Lance has no v1 reader, so a Lance query must reach v2 +// whatever the session says, as ADBC and fluss do. Loads choose by format instead of the flag, +// and Lance is not a load format, so a Lance load never selects v2. +TEST(FileScannerV2Test, LanceQueriesAlwaysUseScannerV2) { + TFileScanRangeParams params; + params.__set_format_type(TFileFormatType::FORMAT_LANCE); + + TQueryOptions disabled; + disabled.__set_enable_file_scanner_v2(false); + EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(disabled, false, params)); + EXPECT_FALSE(FileScanLocalState::TEST_should_use_file_scanner_v2(disabled, true, params)); + + TQueryOptions absent; + EXPECT_TRUE(FileScanLocalState::TEST_should_use_file_scanner_v2(absent, false, params)); +} + TEST(FileScannerV2Test, IcebergOrcDefaultCannotHideDeferredParquetRanges) { TQueryOptions options; options.__set_enable_file_scanner_v2(false); @@ -871,6 +888,7 @@ TEST(FileScannerV2Test, FileFormatConversionMatrix) { {TFileFormatType::FORMAT_JSON, format::FileFormat::JSON}, {TFileFormatType::FORMAT_ARROW, format::FileFormat::ARROW}, {TFileFormatType::FORMAT_WAL, format::FileFormat::WAL}, + {TFileFormatType::FORMAT_LANCE, format::FileFormat::LANCE}, {TFileFormatType::FORMAT_ORC, format::FileFormat::ORC}, }; diff --git a/be/test/format/orc/orc_read_lines.cpp b/be/test/format/orc/orc_read_lines.cpp index 25bffa06e8c12a..ee9daa1b424e3f 100644 --- a/be/test/format/orc/orc_read_lines.cpp +++ b/be/test/format/orc/orc_read_lines.cpp @@ -170,8 +170,8 @@ static void read_orc_line(int64_t line, std::string block_dump, for (auto i = 0; i < row_id_string_column.size(); i++) { GlobalRowLoacationV2 info = *((GlobalRowLoacationV2*)row_id_string_column.get_data_at(i).data); - EXPECT_EQ(info.file_id, 10); - EXPECT_EQ(info.row_id, line); + EXPECT_EQ(info.file_local.file_id, 10); + EXPECT_EQ(info.file_local.row_id, line); EXPECT_EQ(info.backend_id, BackendOptions::get_backend_id()); EXPECT_EQ(info.version, IdManager::ID_VERSION); } diff --git a/be/test/format/parquet/parquet_read_lines.cpp b/be/test/format/parquet/parquet_read_lines.cpp index 2769b94531c85e..66a5b5a166ea5b 100644 --- a/be/test/format/parquet/parquet_read_lines.cpp +++ b/be/test/format/parquet/parquet_read_lines.cpp @@ -184,8 +184,8 @@ static void read_parquet_lines(std::vector numeric_types, for (auto i = 0; i < row_id_string_column.size(); i++) { GlobalRowLoacationV2 info = *((GlobalRowLoacationV2*)row_id_string_column.get_data_at(i).data); - EXPECT_EQ(info.file_id, 10); - EXPECT_EQ(info.row_id, read_lines_tmp.front()); + EXPECT_EQ(info.file_local.file_id, 10); + EXPECT_EQ(info.file_local.row_id, read_lines_tmp.front()); read_lines_tmp.pop_front(); EXPECT_EQ(info.backend_id, BackendOptions::get_backend_id()); EXPECT_EQ(info.version, IdManager::ID_VERSION); diff --git a/be/test/format_v2/lance/lance_nested_null_test.cpp b/be/test/format_v2/lance/lance_nested_null_test.cpp new file mode 100644 index 00000000000000..c36b936de6527a --- /dev/null +++ b/be/test/format_v2/lance/lance_nested_null_test.cpp @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "common/config.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/lance/lance_reader_helper.h" + +namespace doris::format::lance { +namespace { + +class LanceNestedNullTest : public testing::Test { +protected: + void SetUp() override { + _old_validation = config::enable_arrow_input_validation; + config::enable_arrow_input_validation = true; + } + void TearDown() override { config::enable_arrow_input_validation = _old_validation; } + + DataTypePtr null_type() { + return DataTypeFactory::instance().create_data_type(TYPE_NULL, true); + } + + void expect_nulls(const IColumn& column, size_t count) { + const auto& nullable = assert_cast(column); + ASSERT_EQ(count, nullable.size()); + ASSERT_EQ(count, nullable.get_nested_column().size()); + for (size_t row = 0; row < count; ++row) { + EXPECT_TRUE(nullable.is_null_at(row)); + } + } + + void read(const std::shared_ptr& array, const DataTypePtr& type, + MutableColumnPtr* column) { + std::shared_ptr normalized; + ASSERT_TRUE(normalize_lance_arrow_array_for_test(arrow::field("value", array->type()), + array, &normalized, + arrow::default_memory_pool()) + .ok()); + *column = type->create_column(); + ASSERT_TRUE(type->get_serde() + ->read_column_from_arrow(**column, normalized.get(), 0, + normalized->length(), cctz::time_zone {}) + .ok()); + } + +private: + bool _old_validation; +}; + +TEST_F(LanceNestedNullTest, MapsNestedNullSchemas) { + const auto null_field = arrow::field("value", arrow::null()); + const auto schema = arrow::schema({ + arrow::field("list", arrow::list(null_field)), + arrow::field("large_list", arrow::large_list(null_field)), + arrow::field("fixed_list", arrow::fixed_size_list(null_field, 2)), + arrow::field("struct", arrow::struct_({null_field})), + arrow::field("map", arrow::map(arrow::utf8(), null_field)), + arrow::field("nested", arrow::list(arrow::struct_({null_field}))), + arrow::field("list_list", arrow::list(arrow::list(null_field))), + arrow::field("struct_list", + arrow::struct_({arrow::field("list", arrow::list(null_field))})), + arrow::field("list_map", arrow::list(arrow::map(arrow::utf8(), null_field))), + }); + std::vector names; + std::vector types; + ASSERT_TRUE(convert_arrow_schema_to_doris(schema, &names, &types).ok()); + ASSERT_EQ(9, types.size()); + for (const auto& type : types) { + EXPECT_NE(INVALID_TYPE, type->get_primitive_type()); + } + for (size_t i = 0; i < 3; ++i) { + ASSERT_EQ(TYPE_ARRAY, types[i]->get_primitive_type()); + EXPECT_TRUE(assert_cast(*remove_nullable(types[i])) + .get_nested_type() + ->is_null_literal()); + } +} + +TEST_F(LanceNestedNullTest, RejectsNonNullableNullLeaves) { + const auto leaf = arrow::field("item", arrow::null(), false); + for (const auto& type : std::vector> { + arrow::null(), arrow::list(leaf), arrow::large_list(leaf), + arrow::fixed_size_list(leaf, 2), arrow::struct_({leaf}), + arrow::map(arrow::null(), arrow::utf8()), arrow::map(arrow::utf8(), leaf), + arrow::list(arrow::map(arrow::utf8(), leaf))}) { + SCOPED_TRACE(type->ToString()); + std::vector names; + std::vector types; + ASSERT_TRUE(convert_arrow_schema_to_doris( + arrow::schema({arrow::field("value", type, false)}), &names, &types) + .ok()); + ASSERT_EQ(1, types.size()); + EXPECT_EQ(INVALID_TYPE, types[0]->get_primitive_type()); + } +} + +TEST_F(LanceNestedNullTest, PreservesNullTypeInNestedSchemaRpc) { + const auto null = null_type(); + const auto type = make_nullable(std::make_shared( + DataTypes {null, make_nullable(std::make_shared(null)), + make_nullable(std::make_shared( + DataTypeFactory::instance().create_data_type(TYPE_STRING, true), + null))}, + Strings {"scalar", "list", "map"})); + PTypeDesc descriptor; + type->to_protobuf(&descriptor); + ASSERT_EQ(7, descriptor.types_size()); + for (int index : {1, 3, 6}) { + EXPECT_EQ(TPrimitiveType::NULL_TYPE, descriptor.types(index).scalar_type().type()); + } +} + +TEST_F(LanceNestedNullTest, ReadsNullLeafRangesWithoutPhysicalBuffers) { + const auto type = null_type(); + const auto source = std::make_shared(7)->Slice(2, 4); + for (bool validation : {false, true}) { + config::enable_arrow_input_validation = validation; + auto column = type->create_column(); + column->insert_default(); + ASSERT_TRUE( + type->get_serde() + ->read_column_from_arrow(*column, source.get(), 1, 3, cctz::time_zone {}) + .ok()); + expect_nulls(*column, 3); + ASSERT_TRUE( + type->get_serde() + ->read_column_from_arrow(*column, source.get(), 4, 4, cctz::time_zone {}) + .ok()); + expect_nulls(*column, 3); + } +} + +TEST_F(LanceNestedNullTest, RejectsInvalidNullRangesBeforeAppending) { + const auto type = null_type(); + const arrow::NullArray source(3); + for (bool validation : {false, true}) { + config::enable_arrow_input_validation = validation; + auto column = type->create_column(); + column->insert_default(); + for (const auto& [start, end] : + std::vector> {{-1, 1}, {2, 1}, {0, 4}}) { + EXPECT_FALSE(type->get_serde() + ->read_column_from_arrow(*column, &source, start, end, + cctz::time_zone {}) + .ok()); + expect_nulls(*column, 1); + } + } +} + +TEST_F(LanceNestedNullTest, ReadsNullStructParents) { + const std::vector validity {0x05}; + const auto source = std::make_shared( + arrow::struct_({arrow::field("empty", arrow::null())}), 3, + arrow::ArrayVector {std::make_shared(3)}, + arrow::Buffer::Wrap(validity), 1); + const auto type = make_nullable( + std::make_shared(DataTypes {null_type()}, Strings {"empty"})); + MutableColumnPtr column; + read(source->Slice(1, 2), type, &column); + ASSERT_NE(nullptr, column.get()); + const auto& parent = assert_cast(*column); + EXPECT_EQ((NullMap {1, 0}), parent.get_null_map_data()); + const auto& structure = assert_cast(parent.get_nested_column()); + expect_nulls(structure.get_column(0), 2); +} + +TEST_F(LanceNestedNullTest, ReadsNullListElementsAndPreservesParentShape) { + const auto null = null_type(); + const auto type = make_nullable(std::make_shared(null)); + const auto values = std::make_shared(5); + const std::vector offsets {0, 2, 2, 2, 5}; + const std::vector large_offsets {0, 2, 2, 2, 5}; + const std::vector validity {0x0d}; + const std::vector> arrays { + std::make_shared(arrow::list(arrow::null()), 4, + arrow::Buffer::Wrap(offsets), values, + arrow::Buffer::Wrap(validity), 1), + std::make_shared(arrow::large_list(arrow::null()), 4, + arrow::Buffer::Wrap(large_offsets), values, + arrow::Buffer::Wrap(validity), 1), + }; + for (const auto& source : arrays) { + for (bool sliced : {false, true}) { + MutableColumnPtr column; + read(sliced ? source->Slice(1, 3) : source, type, &column); + ASSERT_NE(nullptr, column.get()); + const auto& parent = assert_cast(*column); + EXPECT_EQ((sliced ? NullMap {1, 0, 0} : NullMap {0, 1, 0, 0}), + parent.get_null_map_data()); + const auto& list = assert_cast(parent.get_nested_column()); + EXPECT_EQ((sliced ? ColumnArray::Offsets64 {0, 0, 3} + : ColumnArray::Offsets64 {2, 2, 2, 5}), + list.get_offsets()); + expect_nulls(list.get_data(), sliced ? 3 : 5); + } + } +} + +TEST_F(LanceNestedNullTest, ReadsFixedNullLists) { + const std::vector validity {0x05}; + const auto source = std::make_shared( + arrow::fixed_size_list(arrow::null(), 2), 3, std::make_shared(6), + arrow::Buffer::Wrap(validity), 1); + const auto type = make_nullable(std::make_shared(null_type())); + MutableColumnPtr column; + read(source->Slice(1, 2), type, &column); + ASSERT_NE(nullptr, column.get()); + const auto& parent = assert_cast(*column); + EXPECT_EQ((NullMap {1, 0}), parent.get_null_map_data()); + const auto& list = assert_cast(parent.get_nested_column()); + EXPECT_EQ((ColumnArray::Offsets64 {2, 4}), list.get_offsets()); + expect_nulls(list.get_data(), 4); +} + +TEST_F(LanceNestedNullTest, ReadsNullStructFieldsInsideLists) { + const auto nulls = std::make_shared(3); + const std::vector ids {10, 20, 30}; + const auto id_array = std::make_shared(3, arrow::Buffer::Wrap(ids)); + const auto struct_type = arrow::struct_( + {arrow::field("empty", arrow::null()), arrow::field("id", arrow::int32())}); + const auto structs = std::make_shared(struct_type, 3, + arrow::ArrayVector {nulls, id_array}); + const std::vector offsets {0, 1, 3}; + const auto source = std::make_shared(arrow::list(struct_type), 2, + arrow::Buffer::Wrap(offsets), structs); + const auto doris_struct = make_nullable(std::make_shared( + DataTypes {null_type(), DataTypeFactory::instance().create_data_type(TYPE_INT, true)}, + Strings {"empty", "id"})); + MutableColumnPtr column; + read(source->Slice(1, 1), make_nullable(std::make_shared(doris_struct)), + &column); + ASSERT_NE(nullptr, column.get()); + const auto& list = assert_cast( + assert_cast(*column).get_nested_column()); + const auto& result = assert_cast( + assert_cast(list.get_data()).get_nested_column()); + expect_nulls(result.get_column(0), 2); + const auto& id_column = assert_cast( + assert_cast(result.get_column(1)).get_nested_column()); + EXPECT_EQ((ColumnInt32::Container {20, 30}), id_column.get_data()); +} + +TEST_F(LanceNestedNullTest, ReadsNullMapValues) { + const std::vector keys {10, 20, 30}; + const std::vector offsets {0, 1, 1, 3}; + const std::vector validity {0x05}; + const auto source = std::make_shared( + arrow::map(arrow::int32(), arrow::null()), 3, arrow::Buffer::Wrap(offsets), + std::make_shared(3, arrow::Buffer::Wrap(keys)), + std::make_shared(3), arrow::Buffer::Wrap(validity), 1); + const auto type = make_nullable(std::make_shared( + DataTypeFactory::instance().create_data_type(TYPE_INT, true), null_type())); + MutableColumnPtr column; + read(source->Slice(1, 2), type, &column); + ASSERT_NE(nullptr, column.get()); + const auto& parent = assert_cast(*column); + EXPECT_EQ((NullMap {1, 0}), parent.get_null_map_data()); + const auto& map = assert_cast(parent.get_nested_column()); + EXPECT_EQ((ColumnArray::Offsets64 {0, 2}), map.get_offsets()); + expect_nulls(map.get_values(), 2); + const auto& key_column = assert_cast( + assert_cast(map.get_keys()).get_nested_column()); + EXPECT_EQ((ColumnInt32::Container {20, 30}), key_column.get_data()); +} + +} // namespace +} // namespace doris::format::lance diff --git a/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp b/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp new file mode 100644 index 00000000000000..0fa47687adbff8 --- /dev/null +++ b/be/test/format_v2/lance/lance_runtime_filter_helper_test.cpp @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_runtime_filter_helper.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/field.h" +#include "exprs/create_predicate_function.h" +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vbloom_predicate.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format/format_common.h" + +namespace doris::format::lance { +namespace { + +TExprNode runtime_in_node() { + TExprNode node; + node.__set_type(std::make_shared()->to_thrift()); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + return node; +} + +VExprContextSPtr wrap_runtime_filter(VExprSPtr impl, const TExprNode& node, int filter_id, + bool null_aware = false) { + return VExprContext::create_shared( + RuntimeFilterExpr::create_shared(node, std::move(impl), 0.0, null_aware, filter_id)); +} + +VExprContextSPtr int64_runtime_in(std::string column_name, std::vector values, + int filter_id) { + std::shared_ptr filter(create_set(TYPE_BIGINT, false)); + for (const auto value : values) { + filter->insert(&value); + } + auto node = runtime_in_node(); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return wrap_runtime_filter(std::move(predicate), node, filter_id); +} + +VExprContextSPtr string_runtime_in(std::string column_name, const std::string& value, + int filter_id) { + std::shared_ptr filter(create_set(TYPE_STRING, false)); + StringRef value_ref(value.data(), value.size()); + filter->insert(&value_ref); + auto node = runtime_in_node(); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return wrap_runtime_filter(std::move(predicate), node, filter_id); +} + +template +VExprContextSPtr typed_runtime_in(std::string column_name, + const typename PrimitiveTypeTraits::CppType& value, + DataTypePtr value_type, int filter_id) { + std::shared_ptr filter(create_set(PT, false)); + filter->insert(&value); + auto node = runtime_in_node(); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child( + VSlotRef::create_shared(0, 0, -1, make_nullable(value_type), std::move(column_name))); + return wrap_runtime_filter(std::move(predicate), node, filter_id); +} + +template +VExprContextSPtr typed_runtime_range(std::string column_name, TExprOpcode::type opcode, + const typename PrimitiveTypeTraits::CppType& value, + DataTypePtr value_type, int filter_id, + bool null_aware = false) { + const auto nullable_value_type = make_nullable(value_type); + const auto result_type = make_nullable(std::make_shared()); + + TFunctionName function_name; + function_name.__set_function_name(opcode == TExprOpcode::GE ? "ge" : "le"); + TFunction function; + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + function.__set_arg_types({nullable_value_type->to_thrift(), value_type->to_thrift()}); + function.__set_ret_type(result_type->to_thrift()); + function.__set_has_var_args(false); + + TExprNode predicate_node; + predicate_node.__set_node_type(TExprNodeType::BINARY_PRED); + predicate_node.__set_opcode(opcode); + predicate_node.__set_type(result_type->to_thrift()); + predicate_node.__set_fn(function); + predicate_node.__set_num_children(2); + predicate_node.__set_is_nullable(true); + auto predicate = VectorizedFnCall::create_shared(predicate_node); + predicate->add_child( + VSlotRef::create_shared(0, 0, -1, nullable_value_type, std::move(column_name))); + predicate->add_child(VLiteral::create_shared(value_type, Field::create_field(value))); + + TExprNode wrapper_node; + wrapper_node.__set_type(std::make_shared()->to_thrift()); + wrapper_node.__set_is_nullable(false); + return wrap_runtime_filter(std::move(predicate), wrapper_node, filter_id, null_aware); +} + +VExprContextSPtr int64_runtime_range(std::string column_name, TExprOpcode::type opcode, + int64_t value, int filter_id, bool null_aware = false) { + return typed_runtime_range(std::move(column_name), opcode, value, + std::make_shared(), filter_id, + null_aware); +} + +VExprContextSPtr unsupported_bloom_runtime_filter(std::string column_name, int filter_id) { + auto node = runtime_in_node(); + node.__set_node_type(TExprNodeType::BLOOM_PRED); + node.__set_opcode(TExprOpcode::RT_FILTER); + auto predicate = VBloomPredicate::create_shared(node); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return wrap_runtime_filter(std::move(predicate), node, filter_id); +} + +TEST(LanceRuntimeFilterHelperTest, ConvertsSupportedFiltersToLanceSql) { + const VExprContextSPtrs conjuncts { + int64_runtime_in("order`key", {7}, 3), + string_runtime_in("author", "O'Reilly", 5), + int64_runtime_range("score", TExprOpcode::GE, 10, 7), + int64_runtime_range("score", TExprOpcode::LE, 20, 7), + }; + + const auto schema = arrow::schema({arrow::field("order`key", arrow::int64()), + arrow::field("author", arrow::utf8()), + arrow::field("score", arrow::int64())}); + const auto result = get_or_create_lance_runtime_filter_sql(conjuncts, *schema, nullptr); + ASSERT_NE(result, nullptr); + EXPECT_EQ( + "(`order``key` IN (7)) AND (`author` IN ('O''Reilly')) AND " + "(`score` >= 10) AND (`score` <= 20)", + result->expression); + EXPECT_EQ((std::vector {3, 5, 7}), result->pushable_filter_ids); + EXPECT_TRUE(result->skipped_filter_ids.empty()); +} + +TEST(LanceRuntimeFilterHelperTest, SeparatesSupportedAndUnsupportedRuntimeFilters) { + const VExprContextSPtrs conjuncts { + int64_runtime_in("id", {2}, 3), + unsupported_bloom_runtime_filter("id", 8), + }; + + const auto schema = arrow::schema({arrow::field("id", arrow::int64())}); + const auto result = get_or_create_lance_runtime_filter_sql(conjuncts, *schema, nullptr); + ASSERT_NE(result, nullptr); + EXPECT_EQ("(`id` IN (2))", result->expression); + EXPECT_EQ((std::vector {3}), result->pushable_filter_ids); + EXPECT_EQ((std::vector {8}), result->skipped_filter_ids); +} + +TEST(LanceRuntimeFilterHelperTest, IgnoresNonRuntimeFilterConjuncts) { + const VExprContextSPtrs conjuncts {VExprContext::create_shared(VSlotRef::create_shared( + 0, 0, -1, std::make_shared(), "ordinary_column"))}; + + const auto schema = arrow::schema({arrow::field("ordinary_column", arrow::int64())}); + EXPECT_EQ(nullptr, get_or_create_lance_runtime_filter_sql(conjuncts, *schema, nullptr)); +} + +TEST(LanceRuntimeFilterHelperTest, SkipsNullAwareRangeFilters) { + const VExprContextSPtrs conjuncts { + int64_runtime_range("id", TExprOpcode::GE, 10, 21, true), + int64_runtime_range("id", TExprOpcode::LE, 20, 21, true), + }; + const auto schema = arrow::schema({arrow::field("id", arrow::int64())}); + + const auto result = get_or_create_lance_runtime_filter_sql(conjuncts, *schema, nullptr); + ASSERT_NE(result, nullptr); + EXPECT_TRUE(result->expression.empty()); + EXPECT_TRUE(result->pushable_filter_ids.empty()); + EXPECT_EQ((std::vector {21}), result->skipped_filter_ids); +} + +TEST(LanceRuntimeFilterHelperTest, SkipsStringsThatAreUnsafeAcrossLanceCStringBoundary) { + const VExprContextSPtrs conjuncts { + string_runtime_in("embedded_nul", std::string("a\0b", 3), 22), + string_runtime_in("invalid_utf8", std::string(1, static_cast(0xff)), 23), + }; + const auto schema = arrow::schema({arrow::field("embedded_nul", arrow::utf8()), + arrow::field("invalid_utf8", arrow::utf8())}); + + const auto result = get_or_create_lance_runtime_filter_sql(conjuncts, *schema, nullptr); + ASSERT_NE(result, nullptr); + EXPECT_TRUE(result->expression.empty()); + EXPECT_TRUE(result->pushable_filter_ids.empty()); + EXPECT_EQ((std::vector {22, 23}), result->skipped_filter_ids); +} + +TEST(LanceRuntimeFilterHelperTest, SkipsTimestampNanoRuntimeFilter) { + DateV2Value value; + value.unchecked_set_time(2024, 2, 29, 12, 34, 56, 123456); + const VExprContextSPtrs conjuncts { + typed_runtime_range("timestamp_value", TExprOpcode::LE, value, + std::make_shared(6), 24)}; + const auto nano_schema = arrow::schema( + {arrow::field("timestamp_value", arrow::timestamp(arrow::TimeUnit::NANO))}); + const auto micro_schema = arrow::schema( + {arrow::field("timestamp_value", arrow::timestamp(arrow::TimeUnit::MICRO))}); + + const auto skipped = get_or_create_lance_runtime_filter_sql(conjuncts, *nano_schema, nullptr); + ASSERT_NE(skipped, nullptr); + EXPECT_TRUE(skipped->expression.empty()); + EXPECT_EQ((std::vector {24}), skipped->skipped_filter_ids); + + const auto pushed = get_or_create_lance_runtime_filter_sql(conjuncts, *micro_schema, nullptr); + ASSERT_NE(pushed, nullptr); + EXPECT_EQ("(`timestamp_value` <= TIMESTAMP '2024-02-29 12:34:56.123456')", pushed->expression); +} + +TEST(LanceRuntimeFilterHelperTest, SkipsPhysicalNumericTypesUnsupportedByPinnedPlanner) { + const auto wide_uint64 = static_cast(std::numeric_limits::max()) + 1; + const VExprContextSPtrs conjuncts { + typed_runtime_in("half_float", 1.5F, std::make_shared(), + 25), + typed_runtime_in("scaled_decimal", Decimal32(12345), + std::make_shared(9, 2), 26), + typed_runtime_in("wide_decimal", Decimal128V3(wide_uint64), + std::make_shared(38, 0), 27), + typed_runtime_in("wide_uint64", wide_uint64, + std::make_shared(), 28), + }; + const auto schema = arrow::schema({ + arrow::field("half_float", arrow::float16()), + arrow::field("scaled_decimal", arrow::decimal128(9, 2)), + arrow::field("wide_decimal", arrow::decimal128(38, 0)), + arrow::field("wide_uint64", arrow::uint64()), + }); + + const auto result = get_or_create_lance_runtime_filter_sql(conjuncts, *schema, nullptr); + ASSERT_NE(result, nullptr); + EXPECT_TRUE(result->expression.empty()); + EXPECT_TRUE(result->pushable_filter_ids.empty()); + EXPECT_EQ((std::vector {25, 26, 27, 28}), result->skipped_filter_ids); +} + +TEST(LanceRuntimeFilterHelperTest, KeepsFloatingMinFiltersResidualOnly) { + const auto schema = arrow::schema( + {arrow::field("f32", arrow::float32()), arrow::field("f64", arrow::float64())}); + + // Lance's totalOrder sorts a negative NaN below 1, while Doris keeps it for `x >= 1`. + const VExprContextSPtrs min_filters { + typed_runtime_range("f32", TExprOpcode::GE, 1.0F, + std::make_shared(), 31), + typed_runtime_range("f64", TExprOpcode::GE, 1.0, + std::make_shared(), 32), + }; + const auto skipped = get_or_create_lance_runtime_filter_sql(min_filters, *schema, nullptr); + ASSERT_NE(skipped, nullptr); + EXPECT_TRUE(skipped->expression.empty()); + EXPECT_TRUE(skipped->pushable_filter_ids.empty()); + EXPECT_EQ((std::vector {31, 32}), skipped->skipped_filter_ids); + + // `<=` can only let negative NaNs through to the residual, and IN never matches a NaN row. + const VExprContextSPtrs max_and_in_filters { + typed_runtime_range("f32", TExprOpcode::LE, 1.0F, + std::make_shared(), 33), + typed_runtime_in("f32", 1.5F, std::make_shared(), 34), + }; + const auto pushed = + get_or_create_lance_runtime_filter_sql(max_and_in_filters, *schema, nullptr); + ASSERT_NE(pushed, nullptr); + EXPECT_EQ("(`f32` <= 1) AND (`f32` IN (1.5))", pushed->expression); + EXPECT_EQ((std::vector {33, 34}), pushed->pushable_filter_ids); + EXPECT_TRUE(pushed->skipped_filter_ids.empty()); + + // The two halves of a MIN_MAX filter share one id; only its `<=` half reaches Lance. + const VExprContextSPtrs min_max_filter { + typed_runtime_range("f64", TExprOpcode::GE, 1.0, + std::make_shared(), 35), + typed_runtime_range("f64", TExprOpcode::LE, 2.0, + std::make_shared(), 35), + }; + const auto min_max = get_or_create_lance_runtime_filter_sql(min_max_filter, *schema, nullptr); + ASSERT_NE(min_max, nullptr); + EXPECT_EQ("(`f64` <= 2)", min_max->expression); + EXPECT_EQ((std::vector {35}), min_max->pushable_filter_ids); + EXPECT_TRUE(min_max->skipped_filter_ids.empty()); +} + +TEST(LanceRuntimeFilterHelperTest, ReusesSnapshotAcrossParallelReaders) { + ShardedKVCache cache(2); + const VExprContextSPtrs first_conjuncts { + int64_runtime_in("id", {2}, 12), + int64_runtime_range("score", TExprOpcode::GE, 10, 13), + }; + const VExprContextSPtrs reordered_conjuncts { + int64_runtime_range("score", TExprOpcode::GE, 10, 13), + int64_runtime_in("id", {2}, 12), + }; + + const auto schema = arrow::schema( + {arrow::field("id", arrow::int64()), arrow::field("score", arrow::int64())}); + const auto first = get_or_create_lance_runtime_filter_sql(first_conjuncts, *schema, &cache); + const auto reused = + get_or_create_lance_runtime_filter_sql(reordered_conjuncts, *schema, &cache); + ASSERT_NE(first, nullptr); + ASSERT_NE(reused, nullptr); + EXPECT_EQ(first.get(), reused.get()); + EXPECT_EQ("(`id` IN (2)) AND (`score` >= 10)", reused->expression); + + const auto different = get_or_create_lance_runtime_filter_sql({int64_runtime_in("id", {2}, 14)}, + *schema, &cache); + ASSERT_NE(different, nullptr); + EXPECT_NE(first.get(), different.get()); +} + +} // namespace +} // namespace doris::format::lance diff --git a/be/test/format_v2/lance/lance_session_manager_test.cpp b/be/test/format_v2/lance/lance_session_manager_test.cpp new file mode 100644 index 00000000000000..5672913245a5a5 --- /dev/null +++ b/be/test/format_v2/lance/lance_session_manager_test.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_session_manager.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/metrics/doris_metrics.h" +#include "common/metrics/metrics.h" + +namespace doris::format::lance { +namespace { + +using LanceDatasetPtr = std::unique_ptr; + +std::filesystem::path lance_fixture_path() { + // __FILE__ is relative under -ffile-prefix-map; UTs run from the repository root. + return "./be/test/format_v2/table/lance/data/all_types.lance"; +} + +std::filesystem::path unique_cache_path() { + const auto suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + return std::filesystem::temp_directory_path() / + ("doris_lance_foyer_cache_" + std::to_string(suffix)); +} + +TEST(LanceSessionManagerTest, SessionAndDataCacheConfigurationsAreIndependent) { + LanceSessionManager::Config config { + .lance_index_cache_size_bytes = 0, + .lance_metadata_cache_size_bytes = 0, + .enable_lance_data_cache = false, + // These are deliberately invalid and must be ignored while the data cache is off. + .lance_data_cache_path = "", + .lance_data_cache_disk_capacity_bytes = -1, + .lance_data_cache_read_block_size_bytes = -1, + }; + LanceSessionManager manager(std::move(config)); + LanceDataset* raw_dataset = nullptr; + ASSERT_TRUE(manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset).ok()); + LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); + ASSERT_NE(dataset, nullptr); +} + +TEST(LanceSessionManagerTest, PublishesSessionCacheMetrics) { + constexpr int64_t INDEX_CACHE_CAPACITY = 8 * 1024 * 1024; + constexpr int64_t METADATA_CACHE_CAPACITY = 4 * 1024 * 1024; + LanceSessionManager::Config config { + .lance_index_cache_size_bytes = INDEX_CACHE_CAPACITY, + .lance_metadata_cache_size_bytes = METADATA_CACHE_CAPACITY, + .enable_lance_data_cache = false, + .lance_data_cache_path = "", + .lance_data_cache_disk_capacity_bytes = 0, + .lance_data_cache_read_block_size_bytes = 0, + }; + LanceSessionManager manager(std::move(config)); + LanceDataset* raw_dataset = nullptr; + ASSERT_TRUE(manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset).ok()); + LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); + ASSERT_NE(dataset, nullptr); + + auto* entity = DorisMetrics::instance()->server_entity(); + const auto gauge = [entity](const char* name) { + return static_cast(entity->get_metric(name)); + }; + auto* index_capacity = gauge("lance_session_index_cache_capacity_bytes"); + auto* metadata_capacity = gauge("lance_session_metadata_cache_capacity_bytes"); + ASSERT_NE(index_capacity, nullptr); + ASSERT_NE(metadata_capacity, nullptr); + EXPECT_EQ(index_capacity->value(), INDEX_CACHE_CAPACITY); + EXPECT_EQ(metadata_capacity->value(), METADATA_CACHE_CAPACITY); + EXPECT_NE(gauge("lance_session_index_cache_usage_bytes"), nullptr); + EXPECT_NE(gauge("lance_session_index_cache_entries"), nullptr); + EXPECT_NE(gauge("lance_session_metadata_cache_usage_bytes"), nullptr); + EXPECT_NE(gauge("lance_session_metadata_cache_entries"), nullptr); + EXPECT_NE(entity->get_metric("lance_session_index_cache_hits_total"), nullptr); + EXPECT_NE(entity->get_metric("lance_session_index_cache_misses_total"), nullptr); + EXPECT_NE(entity->get_metric("lance_session_metadata_cache_hits_total"), nullptr); + EXPECT_NE(entity->get_metric("lance_session_metadata_cache_misses_total"), nullptr); +} + +TEST(LanceSessionManagerTest, CreatesFoyerBackedSession) { + const auto cache_path = unique_cache_path(); + std::filesystem::create_directories(cache_path); + const auto cleanup = [&cache_path] { + std::error_code error; + std::filesystem::remove_all(cache_path, error); + }; + + LanceSessionManager::Config config { + .lance_index_cache_size_bytes = 0, + .lance_metadata_cache_size_bytes = 0, + .enable_lance_data_cache = true, + .lance_data_cache_path = cache_path.string(), + .lance_data_cache_disk_capacity_bytes = 32 * 1024 * 1024, + .lance_data_cache_read_block_size_bytes = 64 * 1024, + }; + { + LanceSessionManager manager(std::move(config)); + LanceDataset* raw_dataset = nullptr; + const auto status = + manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, &raw_dataset); + LanceDatasetPtr dataset(raw_dataset, lance_dataset_close); + EXPECT_TRUE(status.ok()) << status; + EXPECT_NE(dataset, nullptr); + } + cleanup(); +} + +} // namespace +} // namespace doris::format::lance diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index a2cb3123af7086..e79d13a916bb2e 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -5412,8 +5412,8 @@ TEST_F(NewOrcReaderTest, ReadGlobalRowIdVirtualColumn) { std::memcpy(&location, rowid.data, sizeof(location)); EXPECT_EQ(location.version, context.version); EXPECT_EQ(location.backend_id, context.backend_id); - EXPECT_EQ(location.file_id, context.file_id); - EXPECT_EQ(location.row_id, row); + EXPECT_EQ(location.file_local.file_id, context.file_id); + EXPECT_EQ(location.file_local.row_id, row); } } diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 37deb32d5bf37b..fde32ee81ad025 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -520,10 +520,10 @@ TEST(ParquetVirtualColumnReaderTest, GlobalRowIdReadSkipSelectAndInvalidArgs) { const auto first = decode_rowid(strings, 0); EXPECT_EQ(first.version, context.version); EXPECT_EQ(first.backend_id, context.backend_id); - EXPECT_EQ(first.file_id, context.file_id); - EXPECT_EQ(first.row_id, 10); - EXPECT_EQ(decode_rowid(strings, 1).row_id, 11); - EXPECT_EQ(decode_rowid(strings, 2).row_id, 14); + EXPECT_EQ(first.file_local.file_id, context.file_id); + EXPECT_EQ(first.file_local.row_id, 10); + EXPECT_EQ(decode_rowid(strings, 1).file_local.row_id, 11); + EXPECT_EQ(decode_rowid(strings, 2).file_local.row_id, 14); GlobalRowIdColumnReader select_reader(context, 20); SelectionVector selection(2); @@ -533,8 +533,8 @@ TEST(ParquetVirtualColumnReaderTest, GlobalRowIdReadSkipSelectAndInvalidArgs) { ASSERT_TRUE(select_reader.select(selection, 2, 5, selected_column).ok()); const auto& selected_strings = assert_cast(*selected_column); ASSERT_EQ(selected_strings.size(), 2); - EXPECT_EQ(decode_rowid(selected_strings, 0).row_id, 21); - EXPECT_EQ(decode_rowid(selected_strings, 1).row_id, 23); + EXPECT_EQ(decode_rowid(selected_strings, 0).file_local.row_id, 21); + EXPECT_EQ(decode_rowid(selected_strings, 1).file_local.row_id, 23); MutableColumnPtr null_column; EXPECT_FALSE(reader.read(1, null_column, &rows_read).ok()); diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 027f20ae89fe23..4aa64afd2f53d4 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -4210,8 +4210,8 @@ TEST_F(NewParquetReaderTest, GlobalRowIdSchemaAndSelectionUseFileRowPosition) { const auto location = decode_rowid(rowids, row); EXPECT_EQ(location.version, context.version); EXPECT_EQ(location.backend_id, context.backend_id); - EXPECT_EQ(location.file_id, context.file_id); - EXPECT_EQ(location.row_id, static_cast(row + 2)); + EXPECT_EQ(location.file_local.file_id, context.file_id); + EXPECT_EQ(location.file_local.row_id, static_cast(row + 2)); } } diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index c8ac9506387ee4..888e9255ddad2c 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -2518,8 +2518,8 @@ TEST_F(ParquetScanTest, GlobalRowIdUsesFileLocalPositionForScanRange) { const auto location = decode_rowid(rowid_column, row); EXPECT_EQ(location.version, context.version); EXPECT_EQ(location.backend_id, context.backend_id); - EXPECT_EQ(location.file_id, context.file_id); - row_ids.push_back(location.row_id); + EXPECT_EQ(location.file_local.file_id, context.file_id); + row_ids.push_back(location.file_local.row_id); } } diff --git a/be/test/format_v2/table/lance/data/all_types.lance/_transactions/0-2e8be81c-130f-4554-a180-287e92c2b147.txn b/be/test/format_v2/table/lance/data/all_types.lance/_transactions/0-2e8be81c-130f-4554-a180-287e92c2b147.txn new file mode 100644 index 00000000000000..df90ebaf40ec87 Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/_transactions/0-2e8be81c-130f-4554-a180-287e92c2b147.txn differ diff --git a/be/test/format_v2/table/lance/data/all_types.lance/_transactions/1-e24b25eb-06e7-419b-9c18-221403eb2d56.txn b/be/test/format_v2/table/lance/data/all_types.lance/_transactions/1-e24b25eb-06e7-419b-9c18-221403eb2d56.txn new file mode 100644 index 00000000000000..736aa2f3f2341f Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/_transactions/1-e24b25eb-06e7-419b-9c18-221403eb2d56.txn differ diff --git a/be/test/format_v2/table/lance/data/all_types.lance/_transactions/2-fb9b3e03-5fe0-450c-83a9-03188bbf2250.txn b/be/test/format_v2/table/lance/data/all_types.lance/_transactions/2-fb9b3e03-5fe0-450c-83a9-03188bbf2250.txn new file mode 100644 index 00000000000000..333ab4adb9c2b5 Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/_transactions/2-fb9b3e03-5fe0-450c-83a9-03188bbf2250.txn differ diff --git a/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..7ef82bebcc9e82 Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..90de3f8f2e17b3 Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..63211bc6fc61e7 Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/all_types.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/all_types.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..f93d3984472d99 --- /dev/null +++ b/be/test/format_v2/table/lance/data/all_types.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":3} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/all_types.lance/data/10000001000101001000110180a7144d7db2f7c78bf34cc2cf.lance b/be/test/format_v2/table/lance/data/all_types.lance/data/10000001000101001000110180a7144d7db2f7c78bf34cc2cf.lance new file mode 100644 index 00000000000000..0ee8bfd602b955 Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/data/10000001000101001000110180a7144d7db2f7c78bf34cc2cf.lance differ diff --git a/be/test/format_v2/table/lance/data/all_types.lance/data/100100101111110001110101291a4b4f979304cbfebff43a09.lance b/be/test/format_v2/table/lance/data/all_types.lance/data/100100101111110001110101291a4b4f979304cbfebff43a09.lance new file mode 100644 index 00000000000000..8b31f62a7c4f24 Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/data/100100101111110001110101291a4b4f979304cbfebff43a09.lance differ diff --git a/be/test/format_v2/table/lance/data/all_types.lance/data/1011110000010000110100012fbe664385be58923eb65eaa10.lance b/be/test/format_v2/table/lance/data/all_types.lance/data/1011110000010000110100012fbe664385be58923eb65eaa10.lance new file mode 100644 index 00000000000000..b345ea09b6e0f4 Binary files /dev/null and b/be/test/format_v2/table/lance/data/all_types.lance/data/1011110000010000110100012fbe664385be58923eb65eaa10.lance differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/auxiliary.idx b/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/auxiliary.idx new file mode 100644 index 00000000000000..f6ecf98bb6ccd5 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/index.idx b/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/index.idx new file mode 100644 index 00000000000000..d7646e4098cb73 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_indices/9634c881-df64-46a5-95dc-2468928008a5/index.idx differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/0-388c379e-6e57-4a48-a567-603ce868bdea.txn b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/0-388c379e-6e57-4a48-a567-603ce868bdea.txn new file mode 100644 index 00000000000000..6643a15e80abc3 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/0-388c379e-6e57-4a48-a567-603ce868bdea.txn differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/1-831523ce-b62e-46b0-bcd2-501920af09b8.txn b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/1-831523ce-b62e-46b0-bcd2-501920af09b8.txn new file mode 100644 index 00000000000000..949b47f39d9e45 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/1-831523ce-b62e-46b0-bcd2-501920af09b8.txn differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn new file mode 100644 index 00000000000000..afebdade12db8d Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_transactions/2-4546c7a5-c68b-47d3-8f54-1e3036ad6429.txn differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..7ed8e3d926b9f4 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..ba5cc92bc12631 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..8f32f9b807f823 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..f93d3984472d99 --- /dev/null +++ b/be/test/format_v2/table/lance/data/distance_range.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":3} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/data/011111011100000101111111a29c2749f892029e3946d88007.lance b/be/test/format_v2/table/lance/data/distance_range.lance/data/011111011100000101111111a29c2749f892029e3946d88007.lance new file mode 100644 index 00000000000000..52f7dbf29cca84 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/data/011111011100000101111111a29c2749f892029e3946d88007.lance differ diff --git a/be/test/format_v2/table/lance/data/distance_range.lance/data/100110001000101110010011f2593f448ba30e92dc54b90567.lance b/be/test/format_v2/table/lance/data/distance_range.lance/data/100110001000101110010011f2593f448ba30e92dc54b90567.lance new file mode 100644 index 00000000000000..53e1eec28d0ace Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range.lance/data/100110001000101110010011f2593f448ba30e92dc54b90567.lance differ diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/auxiliary.idx b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/auxiliary.idx new file mode 100644 index 00000000000000..3ca57b8d6de34b Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx new file mode 100644 index 00000000000000..446e4f4152bf44 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_indices/76cb567d-f24f-4dd0-85b1-b0f486361b5e/index.idx differ diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/0-ec982602-f60a-425e-8b31-f77179548301.txn b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/0-ec982602-f60a-425e-8b31-f77179548301.txn new file mode 100644 index 00000000000000..72e846c899873d Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/0-ec982602-f60a-425e-8b31-f77179548301.txn differ diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn new file mode 100644 index 00000000000000..57cf3b8d551b5d Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_transactions/1-bd4b52a8-552e-4270-b52f-3f678df92fed.txn differ diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..d3526033b05ce3 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..9aebc63bd809b7 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..218abba1699bfa --- /dev/null +++ b/be/test/format_v2/table/lance/data/distance_range_dot.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":2} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/distance_range_dot.lance/data/110000101000110101111101bccc4b4c19af77197f1a92b69c.lance b/be/test/format_v2/table/lance/data/distance_range_dot.lance/data/110000101000110101111101bccc4b4c19af77197f1a92b69c.lance new file mode 100644 index 00000000000000..b72e061da897d2 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_dot.lance/data/110000101000110101111101bccc4b4c19af77197f1a92b69c.lance differ diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/auxiliary.idx b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/auxiliary.idx new file mode 100644 index 00000000000000..1483b3a7c8ea48 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/index.idx b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/index.idx new file mode 100644 index 00000000000000..79948a808fe9a0 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_indices/16361222-c72d-4428-9826-beeb3bfd3667/index.idx differ diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn new file mode 100644 index 00000000000000..6cd36ac96683b0 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/0-2dfc92d6-8481-4406-8893-9fc051e167d2.txn differ diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/1-f4a6b175-3626-4561-817e-c45c857597fa.txn b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/1-f4a6b175-3626-4561-817e-c45c857597fa.txn new file mode 100644 index 00000000000000..b2616f9b6e6246 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_transactions/1-f4a6b175-3626-4561-817e-c45c857597fa.txn differ diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..3a4c998b41263e Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..97194c3229ac31 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..218abba1699bfa --- /dev/null +++ b/be/test/format_v2/table/lance/data/distance_range_sq.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":2} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/distance_range_sq.lance/data/0000100001010101000001014167dc46e9aa87a5e01e036b2a.lance b/be/test/format_v2/table/lance/data/distance_range_sq.lance/data/0000100001010101000001014167dc46e9aa87a5e01e036b2a.lance new file mode 100644 index 00000000000000..ac784824743844 Binary files /dev/null and b/be/test/format_v2/table/lance/data/distance_range_sq.lance/data/0000100001010101000001014167dc46e9aa87a5e01e036b2a.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance new file mode 100644 index 00000000000000..340a57964dc091 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/metadata.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance new file mode 100644 index 00000000000000..9b336efbb78855 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_docs.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance new file mode 100644 index 00000000000000..056f31f2d9346f Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_invert.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance new file mode 100644 index 00000000000000..f265ddd2b8838c Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_indices/59b9c9af-09f0-499a-a888-06036eeec27a/part_0_tokens.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn new file mode 100644 index 00000000000000..65272d53b4f7b4 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/0-4a942168-96d8-4f07-bce0-e3049c8ea6c5.txn differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn new file mode 100644 index 00000000000000..be61bddcadfc85 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/1-b311fe36-9353-4f41-ae19-5054d4a73009.txn differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn new file mode 100644 index 00000000000000..e9216b8131e17a Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_transactions/2-76e69d8c-04b7-402e-8620-7f9a772a2224.txn differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..83edf4ee407b29 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..19ab9f8f79ef41 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..f89780183298b8 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..f93d3984472d99 --- /dev/null +++ b/be/test/format_v2/table/lance/data/fts_indexed.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":3} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance new file mode 100644 index 00000000000000..5079d336a0c6b0 Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/data/000100011001100000000001ee52754ad1a1da89dd0efa4379.lance differ diff --git a/be/test/format_v2/table/lance/data/fts_indexed.lance/data/100101010010010100010000fd779442f589c03379a2858ea8.lance b/be/test/format_v2/table/lance/data/fts_indexed.lance/data/100101010010010100010000fd779442f589c03379a2858ea8.lance new file mode 100644 index 00000000000000..3714a20bae4cfa Binary files /dev/null and b/be/test/format_v2/table/lance/data/fts_indexed.lance/data/100101010010010100010000fd779442f589c03379a2858ea8.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_indices/28f6d79f-9c00-4135-ba4b-fc8de0b635f3/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector.lance/_indices/28f6d79f-9c00-4135-ba4b-fc8de0b635f3/auxiliary.idx new file mode 100644 index 00000000000000..ebb8a2ee93d7c0 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_indices/28f6d79f-9c00-4135-ba4b-fc8de0b635f3/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_indices/28f6d79f-9c00-4135-ba4b-fc8de0b635f3/index.idx b/be/test/format_v2/table/lance/data/multivector.lance/_indices/28f6d79f-9c00-4135-ba4b-fc8de0b635f3/index.idx new file mode 100644 index 00000000000000..ccc24a9818aadc Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_indices/28f6d79f-9c00-4135-ba4b-fc8de0b635f3/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_indices/3b5436e9-4e0b-4db4-abaa-6b34a4ec78ea/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector.lance/_indices/3b5436e9-4e0b-4db4-abaa-6b34a4ec78ea/auxiliary.idx new file mode 100644 index 00000000000000..10691699ec5a6e Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_indices/3b5436e9-4e0b-4db4-abaa-6b34a4ec78ea/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_indices/3b5436e9-4e0b-4db4-abaa-6b34a4ec78ea/index.idx b/be/test/format_v2/table/lance/data/multivector.lance/_indices/3b5436e9-4e0b-4db4-abaa-6b34a4ec78ea/index.idx new file mode 100644 index 00000000000000..92e395ccdb825e Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_indices/3b5436e9-4e0b-4db4-abaa-6b34a4ec78ea/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_indices/54a5446e-5740-4741-9a84-2c3661ca9dac/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector.lance/_indices/54a5446e-5740-4741-9a84-2c3661ca9dac/auxiliary.idx new file mode 100644 index 00000000000000..517044c3b9810c Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_indices/54a5446e-5740-4741-9a84-2c3661ca9dac/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_indices/54a5446e-5740-4741-9a84-2c3661ca9dac/index.idx b/be/test/format_v2/table/lance/data/multivector.lance/_indices/54a5446e-5740-4741-9a84-2c3661ca9dac/index.idx new file mode 100644 index 00000000000000..8a0617ef164202 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_indices/54a5446e-5740-4741-9a84-2c3661ca9dac/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_indices/a8529803-c50d-4ea3-9fcc-302b385b4dc5/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector.lance/_indices/a8529803-c50d-4ea3-9fcc-302b385b4dc5/auxiliary.idx new file mode 100644 index 00000000000000..979f581fd3bcde Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_indices/a8529803-c50d-4ea3-9fcc-302b385b4dc5/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_indices/a8529803-c50d-4ea3-9fcc-302b385b4dc5/index.idx b/be/test/format_v2/table/lance/data/multivector.lance/_indices/a8529803-c50d-4ea3-9fcc-302b385b4dc5/index.idx new file mode 100644 index 00000000000000..25a95be9248b93 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_indices/a8529803-c50d-4ea3-9fcc-302b385b4dc5/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_transactions/0-98248c30-74fc-4328-955b-93ad3a6572c5.txn b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/0-98248c30-74fc-4328-955b-93ad3a6572c5.txn new file mode 100644 index 00000000000000..4315aa008b789e Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/0-98248c30-74fc-4328-955b-93ad3a6572c5.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_transactions/1-2bdf0c85-4f23-47ac-b2c4-ae0bc3ba00e2.txn b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/1-2bdf0c85-4f23-47ac-b2c4-ae0bc3ba00e2.txn new file mode 100644 index 00000000000000..209026c88bf7c9 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/1-2bdf0c85-4f23-47ac-b2c4-ae0bc3ba00e2.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_transactions/2-2f0814cd-23f1-409f-97a7-f11802ae0000.txn b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/2-2f0814cd-23f1-409f-97a7-f11802ae0000.txn new file mode 100644 index 00000000000000..1e08ffe29a2912 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/2-2f0814cd-23f1-409f-97a7-f11802ae0000.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_transactions/3-85968aa2-1806-42ac-8528-a49e4ac9f753.txn b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/3-85968aa2-1806-42ac-8528-a49e4ac9f753.txn new file mode 100644 index 00000000000000..b957684cfc12b3 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/3-85968aa2-1806-42ac-8528-a49e4ac9f753.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_transactions/4-f91c85f5-89bd-4744-9cee-4386e21a8734.txn b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/4-f91c85f5-89bd-4744-9cee-4386e21a8734.txn new file mode 100644 index 00000000000000..e2019da90a8ddc Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/4-f91c85f5-89bd-4744-9cee-4386e21a8734.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_transactions/5-8f686607-2ffa-4f37-b5ba-e9afa8748b3b.txn b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/5-8f686607-2ffa-4f37-b5ba-e9afa8748b3b.txn new file mode 100644 index 00000000000000..e9209d2d67082e Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_transactions/5-8f686607-2ffa-4f37-b5ba-e9afa8748b3b.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551609.manifest b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551609.manifest new file mode 100644 index 00000000000000..14e835db155d72 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551609.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551610.manifest b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551610.manifest new file mode 100644 index 00000000000000..4a3d5af719d302 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551610.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551611.manifest b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551611.manifest new file mode 100644 index 00000000000000..3d7267e59fe23a Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551611.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..b96d93f5e926b4 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..fec3c5a25e733c Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..d3a744c6fb7d9f Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/multivector.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..e84fbad15deb85 --- /dev/null +++ b/be/test/format_v2/table/lance/data/multivector.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":6} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/multivector.lance/data/010011011000011101010111d600654aa4bf152143f353ff5a.lance b/be/test/format_v2/table/lance/data/multivector.lance/data/010011011000011101010111d600654aa4bf152143f353ff5a.lance new file mode 100644 index 00000000000000..3cbec0ae78d081 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/data/010011011000011101010111d600654aa4bf152143f353ff5a.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector.lance/data/01101110111001101011010140ce1040debed650850b651125.lance b/be/test/format_v2/table/lance/data/multivector.lance/data/01101110111001101011010140ce1040debed650850b651125.lance new file mode 100644 index 00000000000000..ca647557a3d10c Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector.lance/data/01101110111001101011010140ce1040debed650850b651125.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_transactions/0-0d81521a-dd03-451f-96a6-81f50ff5edec.txn b/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_transactions/0-0d81521a-dd03-451f-96a6-81f50ff5edec.txn new file mode 100644 index 00000000000000..a004ed7cb2dbf6 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_transactions/0-0d81521a-dd03-451f-96a6-81f50ff5edec.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..b50363ce4f75e7 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..491d734467aa3c --- /dev/null +++ b/be/test/format_v2/table/lance/data/multivector_dimensions.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/multivector_dimensions.lance/data/10000010100100010111001031383243dc838910cb0b1e7016.lance b/be/test/format_v2/table/lance/data/multivector_dimensions.lance/data/10000010100100010111001031383243dc838910cb0b1e7016.lance new file mode 100644 index 00000000000000..864ae773d25b03 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_dimensions.lance/data/10000010100100010111001031383243dc838910cb0b1e7016.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_invalid.lance/_transactions/0-efbaaf9c-8337-4ad8-af21-1b1274f3067e.txn b/be/test/format_v2/table/lance/data/multivector_invalid.lance/_transactions/0-efbaaf9c-8337-4ad8-af21-1b1274f3067e.txn new file mode 100644 index 00000000000000..16eeef9b69145c Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_invalid.lance/_transactions/0-efbaaf9c-8337-4ad8-af21-1b1274f3067e.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_invalid.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/multivector_invalid.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..8e64bb51dc1c4f Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_invalid.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_invalid.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/multivector_invalid.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..491d734467aa3c --- /dev/null +++ b/be/test/format_v2/table/lance/data/multivector_invalid.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/multivector_invalid.lance/data/01001111100111101001000111cfaa495a9857e3b9afc0daee.lance b/be/test/format_v2/table/lance/data/multivector_invalid.lance/data/01001111100111101001000111cfaa495a9857e3b9afc0daee.lance new file mode 100644 index 00000000000000..2b8b6b73cf61d3 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_invalid.lance/data/01001111100111101001000111cfaa495a9857e3b9afc0daee.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_transactions/0-d5ed3155-d19a-4ab9-bb2a-9f1aa804ebd9.txn b/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_transactions/0-d5ed3155-d19a-4ab9-bb2a-9f1aa804ebd9.txn new file mode 100644 index 00000000000000..b067c22ae0c25b Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_transactions/0-d5ed3155-d19a-4ab9-bb2a-9f1aa804ebd9.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..6819d668609026 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..491d734467aa3c --- /dev/null +++ b/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/data/0001001110011010001111001c07dd49c690bdc468eac853df.lance b/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/data/0001001110011010001111001c07dd49c690bdc468eac853df.lance new file mode 100644 index 00000000000000..6302f257e505ec Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_invalid_types.lance/data/0001001110011010001111001c07dd49c690bdc468eac853df.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/7fd93781-4cc1-49fb-ac49-292848aca6fd/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/7fd93781-4cc1-49fb-ac49-292848aca6fd/auxiliary.idx new file mode 100644 index 00000000000000..e70fa7efa44a4b Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/7fd93781-4cc1-49fb-ac49-292848aca6fd/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/7fd93781-4cc1-49fb-ac49-292848aca6fd/index.idx b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/7fd93781-4cc1-49fb-ac49-292848aca6fd/index.idx new file mode 100644 index 00000000000000..94fa280cc55d4d Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/7fd93781-4cc1-49fb-ac49-292848aca6fd/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/e6359049-3830-4e8d-93ac-2993bad0837f/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/e6359049-3830-4e8d-93ac-2993bad0837f/auxiliary.idx new file mode 100644 index 00000000000000..d9125abd06f6e8 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/e6359049-3830-4e8d-93ac-2993bad0837f/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/e6359049-3830-4e8d-93ac-2993bad0837f/index.idx b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/e6359049-3830-4e8d-93ac-2993bad0837f/index.idx new file mode 100644 index 00000000000000..6ed1c566a479a3 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_indices/e6359049-3830-4e8d-93ac-2993bad0837f/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/0-b0e5a832-c817-4b1a-b7fa-b1bb52c0eea9.txn b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/0-b0e5a832-c817-4b1a-b7fa-b1bb52c0eea9.txn new file mode 100644 index 00000000000000..2a3e2eaac73ba9 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/0-b0e5a832-c817-4b1a-b7fa-b1bb52c0eea9.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/1-5f2b5ed1-20aa-4902-9e98-08608d0f16a9.txn b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/1-5f2b5ed1-20aa-4902-9e98-08608d0f16a9.txn new file mode 100644 index 00000000000000..bb78ccfd7f61ba Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/1-5f2b5ed1-20aa-4902-9e98-08608d0f16a9.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/2-b1eca4c0-481a-43d9-b1cd-5d9b69e69c94.txn b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/2-b1eca4c0-481a-43d9-b1cd-5d9b69e69c94.txn new file mode 100644 index 00000000000000..d0d386e29bc1c3 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/2-b1eca4c0-481a-43d9-b1cd-5d9b69e69c94.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/3-4c0638b9-b006-43ee-b556-81b6d0172760.txn b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/3-4c0638b9-b006-43ee-b556-81b6d0172760.txn new file mode 100644 index 00000000000000..5693a0421e9ada Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/3-4c0638b9-b006-43ee-b556-81b6d0172760.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/4-626d52d7-f751-41ec-aba6-4c26d7fc984b.txn b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/4-626d52d7-f751-41ec-aba6-4c26d7fc984b.txn new file mode 100644 index 00000000000000..c1b9003ff1edc1 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_transactions/4-626d52d7-f751-41ec-aba6-4c26d7fc984b.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551610.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551610.manifest new file mode 100644 index 00000000000000..12a889877f0740 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551610.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551611.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551611.manifest new file mode 100644 index 00000000000000..4209f83667b98b Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551611.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..0e223cdf125fef Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..5006d0a0023bdd Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..4be1904ca552ba Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..978d5262ed3c5d --- /dev/null +++ b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":5} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/00011101001010101010111193f4f74d719dca683e57a6fe9f.lance b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/00011101001010101010111193f4f74d719dca683e57a6fe9f.lance new file mode 100644 index 00000000000000..ad75116cb9ea02 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/00011101001010101010111193f4f74d719dca683e57a6fe9f.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/0101001000011001110101011d7b8c41c18a187301e7e13396.lance b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/0101001000011001110101011d7b8c41c18a187301e7e13396.lance new file mode 100644 index 00000000000000..170aa4fdb0e165 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/0101001000011001110101011d7b8c41c18a187301e7e13396.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/111011100000010001101010db50ff47508cb03308b653c236.lance b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/111011100000010001101010db50ff47508cb03308b653c236.lance new file mode 100644 index 00000000000000..63cf851c057add Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_flat.lance/data/111011100000010001101010db50ff47508cb03308b653c236.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/a4de8fca-5bd2-4640-ae9f-e03abcc0b552/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/a4de8fca-5bd2-4640-ae9f-e03abcc0b552/auxiliary.idx new file mode 100644 index 00000000000000..4f60549bdc02c4 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/a4de8fca-5bd2-4640-ae9f-e03abcc0b552/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/a4de8fca-5bd2-4640-ae9f-e03abcc0b552/index.idx b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/a4de8fca-5bd2-4640-ae9f-e03abcc0b552/index.idx new file mode 100644 index 00000000000000..58799424beb42d Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/a4de8fca-5bd2-4640-ae9f-e03abcc0b552/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/f243db3b-18a5-4182-8c37-7481361bf628/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/f243db3b-18a5-4182-8c37-7481361bf628/auxiliary.idx new file mode 100644 index 00000000000000..7537b5add2e61e Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/f243db3b-18a5-4182-8c37-7481361bf628/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/f243db3b-18a5-4182-8c37-7481361bf628/index.idx b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/f243db3b-18a5-4182-8c37-7481361bf628/index.idx new file mode 100644 index 00000000000000..72a682cb0233bd Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_indices/f243db3b-18a5-4182-8c37-7481361bf628/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/0-c28d1916-321e-48b7-a496-97da54424c30.txn b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/0-c28d1916-321e-48b7-a496-97da54424c30.txn new file mode 100644 index 00000000000000..8d77cc6b1eb349 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/0-c28d1916-321e-48b7-a496-97da54424c30.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/1-cb87fcf7-76e1-4c7e-865c-a191e2f9180a.txn b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/1-cb87fcf7-76e1-4c7e-865c-a191e2f9180a.txn new file mode 100644 index 00000000000000..aaa973c8fe9525 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/1-cb87fcf7-76e1-4c7e-865c-a191e2f9180a.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/2-b0ac79bc-4068-4cb9-97de-7c27c89e8534.txn b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/2-b0ac79bc-4068-4cb9-97de-7c27c89e8534.txn new file mode 100644 index 00000000000000..1eff6569688e7d Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/2-b0ac79bc-4068-4cb9-97de-7c27c89e8534.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/3-b7e0ec94-3dc6-4fa7-bc0d-ff28f50e359d.txn b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/3-b7e0ec94-3dc6-4fa7-bc0d-ff28f50e359d.txn new file mode 100644 index 00000000000000..5268a7da37c545 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/3-b7e0ec94-3dc6-4fa7-bc0d-ff28f50e359d.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/4-f99159cc-0549-4cc1-9919-fdecb1aa7f3b.txn b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/4-f99159cc-0549-4cc1-9919-fdecb1aa7f3b.txn new file mode 100644 index 00000000000000..75107ba3a44ff1 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_transactions/4-f99159cc-0549-4cc1-9919-fdecb1aa7f3b.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551610.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551610.manifest new file mode 100644 index 00000000000000..95088d16efbb69 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551610.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551611.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551611.manifest new file mode 100644 index 00000000000000..040d0382b64636 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551611.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..b823e7d3ed9e82 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..9ccf2cbd997f7a Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..23a3a3f7b0ffcc Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..978d5262ed3c5d --- /dev/null +++ b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":5} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/00011100100001011000001059c95649db97fb49f2a95dffb0.lance b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/00011100100001011000001059c95649db97fb49f2a95dffb0.lance new file mode 100644 index 00000000000000..ad75116cb9ea02 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/00011100100001011000001059c95649db97fb49f2a95dffb0.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/001110010001001011111111f440e6407e88010674ae90f980.lance b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/001110010001001011111111f440e6407e88010674ae90f980.lance new file mode 100644 index 00000000000000..63cf851c057add Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/001110010001001011111111f440e6407e88010674ae90f980.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/1100100111001100011011014b3ee846deabcc7ffc48955629.lance b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/1100100111001100011011014b3ee846deabcc7ffc48955629.lance new file mode 100644 index 00000000000000..170aa4fdb0e165 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_ivf_pq.lance/data/1100100111001100011011014b3ee846deabcc7ffc48955629.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_indices/b623b1e5-7461-4a30-880f-394fd4b16617/auxiliary.idx b/be/test/format_v2/table/lance/data/multivector_zero.lance/_indices/b623b1e5-7461-4a30-880f-394fd4b16617/auxiliary.idx new file mode 100644 index 00000000000000..62ebae0a24df41 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/_indices/b623b1e5-7461-4a30-880f-394fd4b16617/auxiliary.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_indices/b623b1e5-7461-4a30-880f-394fd4b16617/index.idx b/be/test/format_v2/table/lance/data/multivector_zero.lance/_indices/b623b1e5-7461-4a30-880f-394fd4b16617/index.idx new file mode 100644 index 00000000000000..e08d50ce53ccca Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/_indices/b623b1e5-7461-4a30-880f-394fd4b16617/index.idx differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/0-8e3d8793-6052-45e0-b1ab-43cc9438e25b.txn b/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/0-8e3d8793-6052-45e0-b1ab-43cc9438e25b.txn new file mode 100644 index 00000000000000..0997919da712f4 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/0-8e3d8793-6052-45e0-b1ab-43cc9438e25b.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/1-5ddfb769-46bb-4370-91f7-34cc271f6ff7.txn b/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/1-5ddfb769-46bb-4370-91f7-34cc271f6ff7.txn new file mode 100644 index 00000000000000..55a14c1f7f2ca1 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/1-5ddfb769-46bb-4370-91f7-34cc271f6ff7.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/2-5512f591-b4c5-4159-88b5-8973be1a07ca.txn b/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/2-5512f591-b4c5-4159-88b5-8973be1a07ca.txn new file mode 100644 index 00000000000000..ffcb12a67f10de Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/_transactions/2-5512f591-b4c5-4159-88b5-8973be1a07ca.txn differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..c05e2e7bae9293 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551613.manifest b/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..02d72ed623daa8 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551613.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..3062a10bc90df3 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/latest_version_hint.json b/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..f93d3984472d99 --- /dev/null +++ b/be/test/format_v2/table/lance/data/multivector_zero.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":3} \ No newline at end of file diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/data/001011001101000001100111eeb02d4d5c92691235fbf98a81.lance b/be/test/format_v2/table/lance/data/multivector_zero.lance/data/001011001101000001100111eeb02d4d5c92691235fbf98a81.lance new file mode 100644 index 00000000000000..3d605e551c80d5 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/data/001011001101000001100111eeb02d4d5c92691235fbf98a81.lance differ diff --git a/be/test/format_v2/table/lance/data/multivector_zero.lance/data/10010111100110111101001099e8aa4d8dbb19533ad5d162fe.lance b/be/test/format_v2/table/lance/data/multivector_zero.lance/data/10010111100110111101001099e8aa4d8dbb19533ad5d162fe.lance new file mode 100644 index 00000000000000..f0bd7eb149af45 Binary files /dev/null and b/be/test/format_v2/table/lance/data/multivector_zero.lance/data/10010111100110111101001099e8aa4d8dbb19533ad5d162fe.lance differ diff --git a/be/test/format_v2/table/lance/distance_range_fixture.md b/be/test/format_v2/table/lance/distance_range_fixture.md new file mode 100644 index 00000000000000..d1f3723304b508 --- /dev/null +++ b/be/test/format_v2/table/lance/distance_range_fixture.md @@ -0,0 +1,93 @@ + + +# Distance range fixture + +Synthetic single-vector data generated with `pylance==7.0.0`, using Lance data +format 2.2 and an IVF_FLAT V3 index. No external service is required to read it. +Rows 0–127 are indexed in fragment 0. Rows 128–130 were appended after indexing +in fragment 1. For query `[127, 0, 0]`, range `[1, 4)` returns rows 126 and 128, +which exercises both index coverage and the unindexed tail. + +Regenerate from the repository root: + +```python +import lance +import pyarrow as pa + +path = "be/test/format_v2/table/lance/data/distance_range.lance" +schema = pa.schema([ + pa.field("row_id", pa.int64(), nullable=False), + pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False), +]) + +def batch(ids): + return pa.Table.from_pydict({ + "row_id": list(ids), + "embedding": [[float(i), 0.0, 0.0] for i in ids], + }, schema=schema) + +dataset = lance.write_dataset(batch(range(128)), path, mode="overwrite", + data_storage_version="2.2") +dataset.create_index("embedding", index_type="IVF_FLAT", num_partitions=1, + name="embedding_idx", index_file_version="V3") +lance.write_dataset(batch(range(128, 131)), path, mode="append", + data_storage_version="2.2") +``` + +Delete the previous generated dataset directory before regenerating to keep the +fixture free of obsolete files. Filenames and index UUIDs may change; tests discover +them from the manifest. + +## Range fallback fixtures + +`distance_range_sq.lance` contains vectors `[i, 0, 0]` for integers 0 through 255, +followed by row 256 with `[0.49, 0, 0]`, and an IVF_SQ V3 index. With query +`[0.51, 0, 0]`, row 256 has squared L2 distance approximately `0.0004`, inside +`[0.0001, 0.001)`. Its quantized distance is zero, so applying the lower bound +before refinement incorrectly discards it. + +`distance_range_dot.lance` contains row 0 with `[1, 0, 0]` and 127 rows with +`[0.5, 0, 0]`, and an IVF_FLAT V3 DOT index. Query `[-FLT_MAX, 0, 0]` with only +an inclusive lower bound of `FLT_MAX` must return row 0. Replacing the absent +upper bound with an exclusive `FLT_MAX` incorrectly discards it. + +These fixtures exercise fragment splits without index UUIDs, as planned by the +FE for unsafe indexed range searches, with both `use_index=true` and `false`. +They use the same schema and writer versions as the fixture above: + +```python +from pathlib import Path + +root = Path("be/test/format_v2/table/lance/data") +for name, values, index_type, metric in [ + ("distance_range_sq", [[float(i), 0.0, 0.0] for i in range(256)] + + [[0.49, 0.0, 0.0]], "IVF_SQ", "l2"), + ("distance_range_dot", [[1.0, 0.0, 0.0]] + + [[0.5, 0.0, 0.0] for _ in range(127)], "IVF_FLAT", "dot"), +]: + data = pa.Table.from_pydict({ + "row_id": list(range(len(values))), "embedding": values, + }, schema=schema) + dataset = lance.write_dataset(data, str(root / (name + ".lance")), + data_storage_version="2.2") + dataset.create_index("embedding", index_type=index_type, metric=metric, + num_partitions=1, name="embedding_idx", + index_file_version="V3") +``` diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp new file mode 100644 index 00000000000000..74344542e57041 --- /dev/null +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -0,0 +1,4651 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/lance_reader.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/consts.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_varbinary.h" +#include "exec/common/endian.h" +#include "exprs/create_predicate_function.h" +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/lance/lance_reader_helper.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" +#include "storage/utils.h" +#include "util/defer_op.h" +#include "util/timezone_utils.h" +#include "util/url_coding.h" + +namespace doris::format::lance { +namespace { + +using Columns = std::vector; + +// The BE-wide Lance session reads its configs once, when a test first opens a dataset. The +// default data cache gives Foyer's disk tier 100 GiB in 2 MiB blocks: 51200 files that stay open +// for the rest of the doris_be_test process, enough to exhaust stdio file descriptors on macOS. +// Keep the cache on, but small and under the UT temp dir. +class LanceDataCacheEnvironment : public ::testing::Environment { +public: + void SetUp() override { + config::lance_data_cache_path = "./ut_dir/lance_data_cache"; + config::lance_data_cache_disk_capacity_bytes = 64L * 1024 * 1024; + } +}; + +[[maybe_unused]] ::testing::Environment* const g_lance_data_cache_env = + ::testing::AddGlobalTestEnvironment(new LanceDataCacheEnvironment); + +class FailingResidualPredicate final : public VExpr { +public: + FailingResidualPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError("Lance reader evaluated a scanner residual predicate"); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const std::string _expr_name = "FailingResidualPredicate"; +}; + +struct LanceFixtureInfo { + int64_t version = 0; + std::vector fragment_ids; +}; + +Status get_fixture_info(const std::filesystem::path& dataset_uri, LanceFixtureInfo* info) { + std::unique_ptr dataset( + lance_dataset_open(dataset_uri.c_str(), nullptr, 0), lance_dataset_close); + if (dataset == nullptr) { + return Status::InternalError("Failed to open Lance fixture: {}", dataset_uri.string()); + } + + info->version = static_cast(lance_dataset_version(dataset.get())); + const auto fragment_count = lance_dataset_fragment_count(dataset.get()); + std::vector raw_fragment_ids(fragment_count); + if (lance_dataset_fragment_ids(dataset.get(), raw_fragment_ids.data()) != 0) { + return Status::InternalError("Failed to list Lance fragments from {}", + dataset_uri.string()); + } + info->fragment_ids.clear(); + info->fragment_ids.reserve(raw_fragment_ids.size()); + for (const auto fragment_id : raw_fragment_ids) { + info->fragment_ids.emplace_back(static_cast(fragment_id)); + } + return Status::OK(); +} + +TFileRangeDesc make_lance_range(const std::filesystem::path& dataset_uri, int64_t version, + std::vector fragment_ids) { + TFileRangeDesc range; + range.__set_format_type(TFileFormatType::FORMAT_LANCE); + TLanceFileDesc lance_params; + lance_params.__set_dataset_uri(dataset_uri.string()); + lance_params.__set_version(version); + lance_params.__set_fragment_ids(std::move(fragment_ids)); + TTableFormatFileDesc table_params; + table_params.__set_table_format_type("lance"); + table_params.__set_lance_params(std::move(lance_params)); + range.__set_table_format_params(std::move(table_params)); + return range; +} + +TFileRangeDesc make_latest_lance_range(const std::filesystem::path& dataset_uri) { + auto range = make_lance_range(dataset_uri, 0, {}); + auto& lance_params = range.table_format_params.lance_params; + lance_params.fragment_ids.clear(); + lance_params.__isset.fragment_ids = false; + return range; +} + +ColumnDefinition projected_column(std::string name, DataTypePtr type) { + return { + .identifier = Field::create_field(name), + .name = std::move(name), + .type = std::move(type), + }; +} + +ColumnDefinition projected_column(std::string name, PrimitiveType type, bool nullable) { + return projected_column(std::move(name), + DataTypeFactory::instance().create_data_type(type, nullable)); +} + +VExprContextSPtr create_int64_runtime_in_conjunct(std::string column_name, + const std::vector& values, + int filter_id) { + std::shared_ptr filter(create_set(TYPE_BIGINT, false)); + for (const auto value : values) { + filter->insert(&value); + } + TExprNode node; + node.__set_type(std::make_shared()->to_thrift()); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return VExprContext::create_shared( + RuntimeFilterExpr::create_shared(node, std::move(predicate), 0.0, false, filter_id)); +} + +template +VExprContextSPtr create_typed_runtime_in_conjunct( + std::string column_name, const typename PrimitiveTypeTraits::CppType& value, + DataTypePtr value_type, int filter_id) { + std::shared_ptr filter(create_set(PT, false)); + filter->insert(&value); + TExprNode node; + node.__set_type(std::make_shared()->to_thrift()); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child( + VSlotRef::create_shared(0, 0, -1, make_nullable(value_type), std::move(column_name))); + return VExprContext::create_shared( + RuntimeFilterExpr::create_shared(node, std::move(predicate), 0.0, false, filter_id)); +} + +VExprContextSPtr create_string_runtime_in_conjunct(std::string column_name, + const std::string& value, int filter_id) { + std::shared_ptr filter(create_set(TYPE_STRING, false)); + const StringRef value_ref(value.data(), value.size()); + filter->insert(&value_ref); + TExprNode node; + node.__set_type(std::make_shared()->to_thrift()); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + auto predicate = VDirectInPredicate::create_shared(node, std::move(filter), true); + predicate->add_child(VSlotRef::create_shared( + 0, 0, -1, make_nullable(std::make_shared()), std::move(column_name))); + return VExprContext::create_shared( + RuntimeFilterExpr::create_shared(node, std::move(predicate), 0.0, false, filter_id)); +} + +template +VExprContextSPtr create_runtime_range_conjunct( + std::string column_name, TExprOpcode::type opcode, + const typename PrimitiveTypeTraits::CppType& value, DataTypePtr value_type, + int filter_id, bool null_aware = false) { + const auto nullable_value_type = make_nullable(value_type); + const auto result_type = make_nullable(std::make_shared()); + TFunctionName function_name; + function_name.__set_function_name(opcode == TExprOpcode::GE ? "ge" : "le"); + TFunction function; + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + function.__set_arg_types({nullable_value_type->to_thrift(), value_type->to_thrift()}); + function.__set_ret_type(result_type->to_thrift()); + function.__set_has_var_args(false); + + TExprNode predicate_node; + predicate_node.__set_node_type(TExprNodeType::BINARY_PRED); + predicate_node.__set_opcode(opcode); + predicate_node.__set_type(result_type->to_thrift()); + predicate_node.__set_fn(function); + predicate_node.__set_num_children(2); + predicate_node.__set_is_nullable(true); + auto predicate = VectorizedFnCall::create_shared(predicate_node); + predicate->add_child( + VSlotRef::create_shared(0, 0, -1, nullable_value_type, std::move(column_name))); + predicate->add_child(VLiteral::create_shared(value_type, Field::create_field(value))); + + TExprNode wrapper_node; + wrapper_node.__set_type(std::make_shared()->to_thrift()); + wrapper_node.__set_is_nullable(false); + return VExprContext::create_shared(RuntimeFilterExpr::create_shared( + wrapper_node, std::move(predicate), 0.0, null_aware, filter_id)); +} + +Status write_lance_record_batch(const std::filesystem::path& dataset_uri, + const std::shared_ptr& batch) { + auto batch_reader = arrow::RecordBatchReader::Make({batch}, batch->schema()); + if (!batch_reader.ok()) { + return Status::InternalError("create Arrow record batch reader failed: {}", + batch_reader.status().message()); + } + ArrowArrayStream stream {}; + const auto export_status = + arrow::ExportRecordBatchReader(std::move(batch_reader).ValueUnsafe(), &stream); + if (!export_status.ok()) { + return Status::InternalError("export Arrow record batch reader failed: {}", + export_status.message()); + } + static_cast( + ::lance::Dataset::write(dataset_uri.string(), &stream, ::lance::WriteMode::Create)); + return Status::OK(); +} + +void add_output_columns(Block* block, const Columns& columns) { + for (const auto& column : columns) { + block->insert({column.type->create_column(), column.type, column.name}); + } +} + +Status init_reader(LanceTableReader* reader, const Columns& projected_columns, + RuntimeState* runtime_state, RuntimeProfile* profile, + TFileScanRangeParams* scan_params, VExprContextSPtrs conjuncts = {}) { + return reader->init({ + .projected_columns = projected_columns, + .conjuncts = std::move(conjuncts), + .format = FileFormat::LANCE, + .scan_params = scan_params, + .io_ctx = nullptr, + .runtime_state = runtime_state, + .scanner_profile = profile, + }); +} + +Status convert_record_batch_for_test(RuntimeState* runtime_state, const Columns& projected_columns, + const std::shared_ptr& record_batch, + Block* block, size_t* rows) { + LanceRecordBatchConverter converter; + RETURN_IF_ERROR(converter.init(runtime_state, projected_columns, SearchKind::NORMAL)); + return converter.convert_record_batch_to_block(record_batch, block, std::nullopt, rows); +} + +Status prepare_range(LanceTableReader* reader, TFileRangeDesc range, + std::optional global_rowid_context = std::nullopt) { + // Assign after value initialization so adding optional split state cannot break this fixture's + // designated initializer under -Wmissing-designated-field-initializers. + SplitReadOptions options; + options.all_runtime_filters_applied = true; + options.current_range = std::move(range); + options.current_split_format = FileFormat::LANCE; + options.global_rowid_context = global_rowid_context; + return reader->prepare_split(options); +} + +Status prepare_fixture(LanceTableReader* reader, const std::filesystem::path& dataset_uri, + const LanceFixtureInfo& fixture, std::vector fragment_ids, + std::optional global_rowid_context = std::nullopt) { + return prepare_range(reader, + make_lance_range(dataset_uri, fixture.version, std::move(fragment_ids)), + global_rowid_context); +} + +TFileScanRangeParams make_float32_vector_search_params( + const std::array& query_values, int64_t top_k, int64_t offset, + std::optional filter = std::nullopt) { + std::string encoded_values(query_values.size() * sizeof(float), '\0'); + for (size_t i = 0; i < query_values.size(); ++i) { + LittleEndian::Store32(encoded_values.data() + i * sizeof(float), + std::bit_cast(query_values[i])); + } + + TSearchVector query_vector; + query_vector.__set_element_type(TVectorElementType::FLOAT32); + query_vector.__set_dimension(static_cast(query_values.size())); + query_vector.__set_values(std::move(encoded_values)); + + TVectorSearchParams vector_params; + vector_params.__set_column("embedding"); + vector_params.__set_query_vector(std::move(query_vector)); + vector_params.__set_top_k(top_k); + vector_params.__set_offset(offset); + vector_params.__set_metric(TVectorMetric::L2); + + TExternalSearchQuery query; + query.__set_vector_search(std::move(vector_params)); + TExternalSearchRequest request; + request.__set_schema_version(1); + request.__set_search_query(std::move(query)); + TVectorSearchOptions vector_search_options; + vector_search_options.__set_use_index(false); + request.__set_vector_search_options(std::move(vector_search_options)); + if (filter.has_value()) { + TSearchFilter search_filter; + search_filter.__set_format(TSearchFilterFormat::SQL); + search_filter.__set_payload(*filter); + request.__set_search_filter(std::move(search_filter)); + } + + TLanceScanParams lance_scan_params; + lance_scan_params.__set_external_search_request(std::move(request)); + TFileScanRangeParams scan_params; + scan_params.__set_lance_scan_params(std::move(lance_scan_params)); + return scan_params; +} + +TFileScanRangeParams make_full_text_search_params( + std::string query_text, int64_t top_k, int64_t offset, + TFtsCoverageMode::type coverage_mode = TFtsCoverageMode::STRICT, + std::optional filter = std::nullopt, + TFtsMatchOperator::type match_operator = TFtsMatchOperator::OR, + int32_t max_fuzzy_distance = 0) { + TFullTextSearchParams full_text_params; + full_text_params.__set_column("body"); + full_text_params.__set_query(std::move(query_text)); + full_text_params.__set_top_k(top_k); + full_text_params.__set_offset(offset); + full_text_params.__set_coverage_mode(coverage_mode); + full_text_params.__set_query_type(TFtsQueryType::MATCH); + full_text_params.__set_match_operator(match_operator); + full_text_params.__set_max_fuzzy_distance(max_fuzzy_distance); + + TExternalSearchQuery query; + query.__set_full_text_search(std::move(full_text_params)); + TExternalSearchRequest request; + request.__set_schema_version(1); + request.__set_search_query(std::move(query)); + if (filter.has_value()) { + TSearchFilter search_filter; + search_filter.__set_format(TSearchFilterFormat::SQL); + search_filter.__set_payload(*filter); + request.__set_search_filter(std::move(search_filter)); + } + + TLanceScanParams lance_scan_params; + lance_scan_params.__set_external_search_request(std::move(request)); + TFileScanRangeParams scan_params; + scan_params.__set_lance_scan_params(std::move(lance_scan_params)); + return scan_params; +} + +TFileScanRangeParams make_phrase_search_params( + std::string query_text, int64_t top_k, int64_t offset, int32_t slop, + TFtsCoverageMode::type coverage_mode = TFtsCoverageMode::STRICT, + std::optional filter = std::nullopt) { + auto scan_params = make_full_text_search_params(std::move(query_text), top_k, offset, + coverage_mode, std::move(filter)); + auto& full_text = + scan_params.lance_scan_params.external_search_request.search_query.full_text_search; + full_text.__set_query_type(TFtsQueryType::PHRASE); + full_text.__isset.match_operator = false; + full_text.__isset.max_fuzzy_distance = false; + full_text.__set_phrase_slop(slop); + return scan_params; +} + +Status get_index_segment_uuids(const std::filesystem::path& dataset_uri, const char* index_name, + std::vector* encoded_uuids) { + std::unique_ptr dataset( + lance_dataset_open(dataset_uri.c_str(), nullptr, 0), lance_dataset_close); + if (dataset == nullptr) { + return Status::InternalError("Failed to open Lance fixture: {}", dataset_uri.string()); + } + const auto segment_count = lance_dataset_index_segment_count(dataset.get(), index_name); + if (segment_count == 0) { + return Status::InternalError("Lance fixture index '{}' has no segments", index_name); + } + std::vector raw_uuids(segment_count * 16); + uint64_t actual_count = 0; + if (lance_dataset_index_segments(dataset.get(), index_name, raw_uuids.data(), segment_count, + &actual_count) != 0 || + actual_count != segment_count) { + return Status::InternalError("Failed to enumerate Lance fixture index '{}' segments", + index_name); + } + encoded_uuids->clear(); + encoded_uuids->reserve(segment_count); + for (size_t index = 0; index < segment_count; ++index) { + encoded_uuids->emplace_back(reinterpret_cast(raw_uuids.data() + index * 16), + 16); + } + return Status::OK(); +} + +// A split over `fragment_ids` searching the segments of `index_name`, as FE plans an indexed split; +// BE searches a vector index only through the segments FE names. +TFileRangeDesc make_indexed_lance_range(const std::filesystem::path& dataset_uri, + const LanceFixtureInfo& fixture, + std::vector fragment_ids, const char* index_name) { + auto range = make_lance_range(dataset_uri, fixture.version, std::move(fragment_ids)); + std::vector segment_uuids; + EXPECT_TRUE(get_index_segment_uuids(dataset_uri, index_name, &segment_uuids).ok()); + range.table_format_params.lance_params.__set_index_segment_uuids(std::move(segment_uuids)); + return range; +} + +TFileRangeDesc make_full_text_search_range(const std::filesystem::path& dataset_uri, + const LanceFixtureInfo& fixture, + const char* index_name) { + auto range = make_lance_range(dataset_uri, fixture.version, fixture.fragment_ids); + std::vector segment_uuids; + EXPECT_TRUE(get_index_segment_uuids(dataset_uri, index_name, &segment_uuids).ok()); + range.table_format_params.lance_params.__set_index_segment_uuids(std::move(segment_uuids)); + return range; +} + +TEST(LanceTableReaderVectorSearchTest, RejectsMalformedVectorPayloadBeforeReadingIt) { + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_vector_search_invalid_request"); + auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0); + scan_params.lance_scan_params.external_search_request.search_query.vector_search.query_vector + .__set_dimension(4); + + LanceTableReader reader; + const auto status = init_reader(&reader, columns, &state, &profile, &scan_params); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("query vector byte size"), std::string::npos); +} + +TEST(LanceTableReaderVectorSearchTest, ValidatesMultiVectorWireShapeBeforeDatasetAccess) { + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + auto valid = make_float32_vector_search_params({1.0F, 0.0F, 0.0F}, 4, 0); + auto& request = valid.lance_scan_params.external_search_request; + request.__set_schema_version(1); + auto& query = request.search_query.vector_search.query_vector; + query.__set_dimension(3); + query.__set_num_vectors(1); + const auto check = [&](TFileScanRangeParams params, bool expected) { + RuntimeProfile profile("multi_vector_validation"); + LanceTableReader reader; + EXPECT_EQ(expected, init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + }; + check(valid, true); + auto excessive_work = valid; + excessive_work.lance_scan_params.external_search_request.search_query.vector_search.__set_top_k( + 100001); + check(excessive_work, false); + excessive_work.lance_scan_params.external_search_request.search_query.vector_search.__set_top_k( + 1); + excessive_work.lance_scan_params.external_search_request.vector_search_options + .__set_refine_factor(std::numeric_limits::max()); + check(excessive_work, false); + for (const int count : {0, -1, 2, std::numeric_limits::max()}) { + auto invalid = valid; + invalid.lance_scan_params.external_search_request.search_query.vector_search.query_vector + .__set_num_vectors(count); + check(invalid, false); + } + auto unsupported_version = valid; + unsupported_version.lance_scan_params.external_search_request.__set_schema_version(2); + check(unsupported_version, false); + auto missing_count = valid; + missing_count.lance_scan_params.external_search_request.search_query.vector_search.query_vector + .__isset.num_vectors = false; + // Without num_vectors, the same payload is an ordinary single-vector request. + check(missing_count, true); + auto hamming = valid; + hamming.lance_scan_params.external_search_request.search_query.vector_search.__set_metric( + TVectorMetric::HAMMING); + check(hamming, false); + for (const auto type : {TVectorElementType::INT8, TVectorElementType::UINT8}) { + auto integer = valid; + auto& q = integer.lance_scan_params.external_search_request.search_query.vector_search + .query_vector; + q.__set_element_type(type); + q.__set_values(std::string(3, '\0')); + check(integer, false); + } +} + +TEST(LanceTableReaderFullTextSearchTest, ValidatesRequestAndScoreTypeBeforeDatasetAccess) { + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile valid_profile("lance_fts_valid_request"); + auto valid_params = make_full_text_search_params("lance", 4, 1); + LanceTableReader valid_reader; + ASSERT_TRUE(init_reader(&valid_reader, columns, &state, &valid_profile, &valid_params).ok()); + + RuntimeProfile empty_query_profile("lance_fts_empty_query"); + auto empty_query_params = make_full_text_search_params("", 4, 0); + LanceTableReader empty_query_reader; + const auto empty_query_status = init_reader(&empty_query_reader, columns, &state, + &empty_query_profile, &empty_query_params); + EXPECT_FALSE(empty_query_status.ok()); + EXPECT_NE(empty_query_status.to_string().find("non-empty query"), std::string::npos); + + RuntimeProfile coverage_profile("lance_fts_invalid_coverage"); + auto invalid_coverage_params = + make_full_text_search_params("lance", 4, 0, static_cast(99)); + LanceTableReader coverage_reader; + const auto coverage_status = init_reader(&coverage_reader, columns, &state, &coverage_profile, + &invalid_coverage_params); + EXPECT_FALSE(coverage_status.ok()); + EXPECT_NE(coverage_status.to_string().find("STRICT or INDEX_ONLY"), std::string::npos); + + const Columns wrong_score_columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_DOUBLE, true), + }; + RuntimeProfile score_profile("lance_fts_invalid_score_type"); + auto score_params = make_full_text_search_params("lance", 4, 0); + LanceTableReader score_reader; + const auto score_status = + init_reader(&score_reader, wrong_score_columns, &state, &score_profile, &score_params); + EXPECT_FALSE(score_status.ok()); + EXPECT_NE(score_status.to_string().find("must have Doris FLOAT type"), std::string::npos); +} + +TEST(LanceTableReaderFullTextSearchTest, ValidatesQuerySpecificParameters) { + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + + RuntimeProfile phrase_profile("lance_fts_phrase_request"); + auto phrase_params = make_phrase_search_params("lance search", 4, 0, 1); + LanceTableReader phrase_reader; + ASSERT_TRUE(init_reader(&phrase_reader, columns, &state, &phrase_profile, &phrase_params).ok()); + + RuntimeProfile fuzzy_profile("lance_fts_fuzzy_request"); + auto fuzzy_params = make_full_text_search_params("lance", 4, 0, TFtsCoverageMode::STRICT, + std::nullopt, TFtsMatchOperator::OR, 1); + LanceTableReader fuzzy_reader; + const auto fuzzy_status = + init_reader(&fuzzy_reader, columns, &state, &fuzzy_profile, &fuzzy_params); + EXPECT_FALSE(fuzzy_status.ok()); + EXPECT_NE(fuzzy_status.to_string().find("does not yet support max_fuzzy_distance=1"), + std::string::npos); + + RuntimeProfile match_slop_profile("lance_fts_match_with_slop"); + auto match_slop_params = make_full_text_search_params("lance", 4, 0); + match_slop_params.lance_scan_params.external_search_request.search_query.full_text_search + .__set_phrase_slop(1); + LanceTableReader match_slop_reader; + const auto match_slop_status = init_reader(&match_slop_reader, columns, &state, + &match_slop_profile, &match_slop_params); + EXPECT_FALSE(match_slop_status.ok()); + EXPECT_NE(match_slop_status.to_string().find("MATCH query cannot set phrase_slop"), + std::string::npos); + + RuntimeProfile negative_slop_profile("lance_fts_negative_phrase_slop"); + auto negative_slop_params = make_phrase_search_params("lance search", 4, 0, -1); + LanceTableReader negative_slop_reader; + const auto negative_slop_status = init_reader(&negative_slop_reader, columns, &state, + &negative_slop_profile, &negative_slop_params); + EXPECT_FALSE(negative_slop_status.ok()); + EXPECT_NE(negative_slop_status.to_string().find("phrase_slop must be non-negative"), + std::string::npos); +} + +std::vector> read_full_text_search_rows(LanceTableReader* reader, + Block* block) { + std::vector> rows; + bool eos = false; + while (!eos) { + EXPECT_TRUE(reader->get_block(block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block->get_by_position(0).column); + const auto& scores = assert_cast(*block->get_by_position(1).column); + const auto& score_values = assert_cast(scores.get_nested_column()); + for (size_t row = 0; row < block->rows(); ++row) { + EXPECT_EQ(0, scores.get_null_map_data()[row]); + rows.emplace_back(row_ids.get_data()[row], score_values.get_data()[row]); + } + } + return rows; +} + +TEST(LanceTableReaderFullTextSearchTest, SearchesIndexedSnapshotWithOptionalScoreProjection) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/fts_indexed.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + auto range = make_full_text_search_range(dataset_uri, fixture, "body_fts"); + + const Columns scored_columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile scored_profile("lance_fts_indexed_fixture"); + auto scored_params = make_full_text_search_params("lance", 4, 0); + LanceTableReader scored_reader; + ASSERT_TRUE(init_reader(&scored_reader, scored_columns, &state, &scored_profile, &scored_params) + .ok()); + ASSERT_TRUE(prepare_range(&scored_reader, range).ok()); + Block scored_block; + add_output_columns(&scored_block, scored_columns); + const auto scored_rows = read_full_text_search_rows(&scored_reader, &scored_block); + ASSERT_EQ(4U, scored_rows.size()); + EXPECT_EQ((std::vector {3, 2, 1, 7}), + (std::vector {scored_rows[0].first, scored_rows[1].first, + scored_rows[2].first, scored_rows[3].first})); + for (size_t index = 0; index < scored_rows.size(); ++index) { + EXPECT_GT(scored_rows[index].second, 0.0F); + if (index > 0) { + EXPECT_GT(scored_rows[index - 1].second, scored_rows[index].second); + } + } + EXPECT_TRUE(scored_reader.close().ok()); + + // Lance returns _score for ordering even when Doris does not materialize it. The reader must + // ignore that generated column while still rejecting any unexpected _rowid output. + const Columns row_id_columns {projected_column("row_id", TYPE_BIGINT, false)}; + RuntimeProfile row_id_profile("lance_fts_without_score_projection_fixture"); + auto row_id_params = make_full_text_search_params("lance", 2, 0); + LanceTableReader row_id_reader; + ASSERT_TRUE(init_reader(&row_id_reader, row_id_columns, &state, &row_id_profile, &row_id_params) + .ok()); + ASSERT_TRUE(prepare_range(&row_id_reader, range).ok()); + Block row_id_block; + add_output_columns(&row_id_block, row_id_columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(row_id_reader.get_block(&row_id_block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*row_id_block.get_by_position(0).column); + row_ids.insert(row_ids.end(), values.get_data().begin(), values.get_data().end()); + } + } + EXPECT_EQ((std::vector {3, 2}), row_ids); + EXPECT_TRUE(row_id_reader.close().ok()); +} + +TEST(LanceTableReaderFullTextSearchTest, SupportsMatchAndPhraseQueries) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/fts_indexed.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + auto range = make_full_text_search_range(dataset_uri, fixture, "body_fts"); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + + RuntimeProfile match_and_profile("lance_fts_match_and"); + auto match_and_params = make_full_text_search_params( + "lance storage", 10, 0, TFtsCoverageMode::STRICT, std::nullopt, TFtsMatchOperator::AND); + LanceTableReader match_and_reader; + ASSERT_TRUE( + init_reader(&match_and_reader, columns, &state, &match_and_profile, &match_and_params) + .ok()); + ASSERT_TRUE(prepare_range(&match_and_reader, range).ok()); + Block match_and_block; + add_output_columns(&match_and_block, columns); + const auto match_and_rows = read_full_text_search_rows(&match_and_reader, &match_and_block); + ASSERT_EQ(1U, match_and_rows.size()); + EXPECT_EQ(7, match_and_rows[0].first); + EXPECT_TRUE(match_and_reader.close().ok()); + + RuntimeProfile phrase_profile("lance_fts_phrase_exact"); + auto phrase_params = make_phrase_search_params("lance search", 10, 0, 0); + LanceTableReader phrase_reader; + ASSERT_TRUE(init_reader(&phrase_reader, columns, &state, &phrase_profile, &phrase_params).ok()); + ASSERT_TRUE(prepare_range(&phrase_reader, range).ok()); + Block phrase_block; + add_output_columns(&phrase_block, columns); + const auto phrase_rows = read_full_text_search_rows(&phrase_reader, &phrase_block); + std::vector phrase_row_ids; + phrase_row_ids.reserve(phrase_rows.size()); + for (const auto& [row_id, score] : phrase_rows) { + EXPECT_GT(score, 0.0F); + phrase_row_ids.emplace_back(row_id); + } + std::sort(phrase_row_ids.begin(), phrase_row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3}), phrase_row_ids); + EXPECT_TRUE(phrase_reader.close().ok()); + + RuntimeProfile phrase_slop_profile("lance_fts_phrase_slop"); + auto phrase_slop_params = make_phrase_search_params("lance engine", 10, 0, 1); + LanceTableReader phrase_slop_reader; + ASSERT_TRUE(init_reader(&phrase_slop_reader, columns, &state, &phrase_slop_profile, + &phrase_slop_params) + .ok()); + ASSERT_TRUE(prepare_range(&phrase_slop_reader, range).ok()); + Block phrase_slop_block; + add_output_columns(&phrase_slop_block, columns); + const auto phrase_slop_rows = + read_full_text_search_rows(&phrase_slop_reader, &phrase_slop_block); + std::vector phrase_slop_row_ids; + phrase_slop_row_ids.reserve(phrase_slop_rows.size()); + for (const auto& [row_id, score] : phrase_slop_rows) { + EXPECT_GT(score, 0.0F); + phrase_slop_row_ids.emplace_back(row_id); + } + std::sort(phrase_slop_row_ids.begin(), phrase_slop_row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3}), phrase_slop_row_ids); + EXPECT_TRUE(phrase_slop_reader.close().ok()); +} + +TEST(LanceTableReaderFullTextSearchTest, SearchFilterHonorsPrefilterFlag) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/fts_indexed.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + auto range = make_full_text_search_range(dataset_uri, fixture, "body_fts"); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_score", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + // "lance" ranks rows 3, 2, 1, 7. With top_k = 1, the default postfilter keeps only the best + // match, row 3, and the filter then removes it; prefilter applies the filter first, so the best + // remaining match, row 2, is returned. + for (const bool prefilter : {false, true}) { + SCOPED_TRACE(prefilter ? "prefilter" : "postfilter"); + RuntimeProfile profile("lance_fts_prefilter_flag"); + auto params = make_full_text_search_params("lance", 1, 0, TFtsCoverageMode::STRICT, + std::string("row_id <= 2")); + if (prefilter) { + params.lance_scan_params.external_search_request.__set_prefilter(true); + } + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + const auto rows = read_full_text_search_rows(&reader, &block); + if (prefilter) { + ASSERT_EQ(1U, rows.size()); + EXPECT_EQ(2, rows[0].first); + } else { + EXPECT_TRUE(rows.empty()); + } + EXPECT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderVectorSearchTest, RejectsMalformedIndexSegmentUuid) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_vector_search_invalid_index_segment_uuid"); + auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_lance_range(dataset_uri, fixture.version, fixture.fragment_ids); + range.table_format_params.lance_params.__set_index_segment_uuids({"too-short"}); + + const auto status = prepare_range(&reader, std::move(range)); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("must contain 16 bytes"), std::string::npos); + EXPECT_TRUE(reader.close().ok()); +} + +std::vector> read_vector_search_rows(LanceTableReader* reader, + Block* block) { + std::vector> rows; + bool eos = false; + while (!eos) { + EXPECT_TRUE(reader->get_block(block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block->get_by_position(0).column); + const auto& distances = + assert_cast(*block->get_by_position(1).column); + const auto& distance_values = + assert_cast(distances.get_nested_column()); + for (size_t row = 0; row < block->rows(); ++row) { + EXPECT_EQ(0, distances.get_null_map_data()[row]); + rows.emplace_back(row_ids.get_data()[row], distance_values.get_data()[row]); + } + } + return rows; +} + +GlobalRowLoacationV2 decode_lance_row_id(const ColumnString& column, size_t row) { + const auto encoded = column.get_data_at(row); + EXPECT_EQ(sizeof(GlobalRowLoacationV2), encoded.size); + GlobalRowLoacationV2 location(ROW_VERSION::LANCE_DATASET_ROW_ID, 0, 0, 0); + if (encoded.size == sizeof(location)) { + std::memcpy(&location, encoded.data, sizeof(location)); + } + return location; +} + +TEST(LanceTableReaderVectorSearchTest, MultiVectorScoresFiltersOffsetsAndIndexedCoverage) { + const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/multivector.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + ASSERT_EQ(2, fixture.fragment_ids.size()); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + TQueryOptions options; + options.__set_batch_size(1); + state.set_query_options(options); + for (const auto type : + {TVectorElementType::FLOAT16, TVectorElementType::FLOAT32, TVectorElementType::FLOAT64}) { + const size_t width = type == TVectorElementType::FLOAT16 ? 2 + : type == TVectorElementType::FLOAT32 ? 4 + : 8; + const std::string column = type == TVectorElementType::FLOAT16 ? "vectors16" + : type == TVectorElementType::FLOAT32 ? "vectors32" + : "vectors64"; + std::string bytes(4 * width, '\0'); + for (const size_t i : {0U, 3U}) { + if (width == 2) LittleEndian::Store16(bytes.data() + i * width, 0x3c00); + if (width == 4) + LittleEndian::Store32(bytes.data() + i * width, std::bit_cast(1.0F)); + if (width == 8) + LittleEndian::Store64(bytes.data() + i * width, std::bit_cast(1.0)); + } + for (const bool indexed : {false, true}) { + for (const bool filtered : {false, true}) { + SCOPED_TRACE(column + " indexed=" + std::to_string(indexed) + + " filtered=" + std::to_string(filtered)); + RuntimeProfile profile("multi_vector_search"); + auto params = make_float32_vector_search_params({0, 0, 0}, 10, 0); + auto& request = params.lance_scan_params.external_search_request; + request.__set_schema_version(1); + request.vector_search_options.__set_use_index(indexed); + request.vector_search_options.__set_nprobes(1); + auto& search = request.search_query.vector_search; + search.__set_column(column); + search.__set_metric(indexed ? TVectorMetric::COSINE : TVectorMetric::L2); + auto& q = search.query_vector; + q.__set_dimension(2); + q.__set_num_vectors(2); + q.__set_element_type(type); + q.__set_values(bytes); + if (filtered) { + TSearchFilter filter; + filter.__set_format(TSearchFilterFormat::SQL); + filter.__set_payload("row_id >= 2"); + request.__set_search_filter(filter); + request.__set_prefilter(true); + search.__set_offset(1); + search.__set_top_k(2); + } + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + // BE searches an index only through the segments FE planned; the index covers + // fragment 0 and fragment 1 is searched flat in the same split. + auto range = make_lance_range(uri, fixture.version, fixture.fragment_ids); + if (indexed) { + std::vector segments; + ASSERT_TRUE(get_index_segment_uuids(uri, (column + "_idx").c_str(), &segments) + .ok()); + range.table_format_params.lance_params.__set_index_segment_uuids(segments); + } + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + auto rows = read_vector_search_rows(&reader, &block); + ASSERT_EQ(filtered ? 2 : 4, rows.size()); + // L2 is sum of per-query nearest squared distances; null/empty rows never rank. + if (!indexed) { + const std::vector> expected = + filtered ? std::vector> {{2, 6.0F}, {3, 8.0F}} + : std::vector> { + {1, 0.0F}, {6, 2.0F}, {2, 6.0F}, {3, 8.0F}}; + EXPECT_EQ(expected, rows); + } else { + std::sort(rows.begin(), rows.end()); + const std::vector> expected = + filtered ? std::vector< + std::pair> {{2, 1.0F}, + {6, + 2.0F - std::sqrt(2.0F)}} + : std::vector> { + {1, 0.0F}, + {2, 1.0F}, + {3, 0.0F}, + {6, 2.0F - std::sqrt(2.0F)}}; + // Row 6 was appended after index creation and must use the same score scale. + for (size_t i = 0; i < rows.size(); ++i) { + EXPECT_EQ(expected[i].first, rows[i].first); + EXPECT_NEAR(expected[i].second, rows[i].second, 1e-5); + } + } + EXPECT_TRUE(reader.close().ok()); + if (indexed) { + // Warm searches still perform scoring even when every index partition is cached. + for (const char* name : + {"LanceIndexPartitionLoadTime", "LanceIndexCpuQueueWaitTime", + "LanceIndexSearchTime", "LanceIndexQueryPrepareTime", + "LanceIndexDistanceTopKTime", "LanceIndexResultMaterializeTime"}) { + auto* counter = profile.get_counter(name); + ASSERT_NE(nullptr, counter) << name; + EXPECT_GT(counter->value(), 0) << name; + } + // Read metrics after close: lance-c publishes its final execution summary + // when the stream is released, including for an early top-k stop. + for (const char* name : {"RowIdPrefilterLoads", "RowIdPrefilterInputRows", + "RowIdPrefilterInputBatches", "RowIdPrefilterIds", + "RowIdPrefilterLoadTime", "RowIdPrefilterInputTime", + "RowIdPrefilterBuildTime"}) { + auto* counter = profile.get_counter(name); + if (filtered) { + ASSERT_NE(nullptr, counter) << name; + EXPECT_GT(counter->value(), 0) << name; + } else if (counter != nullptr) { + EXPECT_EQ(counter->value(), 0) << name; + } + } + } else { + // Flat splits still expose the same metric tree so FE aggregation can merge + // later indexed splits even when this task reports no vector-index metrics. + auto* counter = profile.get_counter("LanceIndexPartitionLoadTime"); + ASSERT_NE(nullptr, counter); + EXPECT_EQ(0, counter->value()); + } + } + } + } +} + +std::vector representative_multi_vector(int id, int subvector) { + std::vector result(128); + for (int j = 0; j < 128; ++j) { + result[j] = ((id * 17 + subvector * 29 + j * 13 + j * j * 7 + id * j * 3) % 1009 - 504) / + 512.0F; + } + return result; +} + +TEST(LanceTableReaderVectorSearchTest, MultiVectorSegmentTopKMatchesIndependentGlobalOracle) { + const auto first = representative_multi_vector(7, 0); + const auto second = representative_multi_vector(11, 1); + std::vector query = first; + query.insert(query.end(), second.begin(), second.end()); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (const auto* kind : {"flat", "pq"}) { + const std::filesystem::path uri = std::string("./be/test/format_v2/table/lance/data/") + + "multivector_ivf_" + kind + ".lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + ASSERT_EQ(3, fixture.fragment_ids.size()); + std::vector segments; + ASSERT_TRUE(get_index_segment_uuids(uri, "vectors_idx", &segments).ok()); + ASSERT_EQ(2, segments.size()); + for (const bool filtered : {false, true}) { + std::vector> oracle; + for (int id = 1; id <= 768; ++id) { + if (filtered && id % 5 != 0) continue; + double score = 0; + for (const auto& q : {first, second}) { + double best = std::numeric_limits::infinity(); + for (int sub = 0; sub < 1 + id % 4; ++sub) { + const auto v = representative_multi_vector(id, sub); + double dot = 0, q_norm = 0, v_norm = 0; + for (size_t j = 0; j < v.size(); ++j) { + dot += static_cast(q[j]) * v[j]; + q_norm += static_cast(q[j]) * q[j]; + v_norm += static_cast(v[j]) * v[j]; + } + best = std::min(best, 1.0 - dot / std::sqrt(q_norm * v_norm)); + } + score += best; + } + oracle.emplace_back(id, static_cast(score)); + } + const auto order = [](const auto& a, const auto& b) { + return std::tie(a.second, a.first) < std::tie(b.second, b.first); + }; + std::sort(oracle.begin(), oracle.end(), order); + for (const int batch_size : {1, 64}) { + TQueryOptions options; + options.__set_batch_size(batch_size); + state.set_query_options(options); + std::vector> candidates; + for (int split = 0; split < 3; ++split) { + SCOPED_TRACE(std::string(kind) + " split=" + std::to_string(split) + + " filtered=" + std::to_string(filtered) + + " batch=" + std::to_string(batch_size)); + RuntimeProfile profile("multi_vector_segment_topk"); + // FE sends top_k + offset to each split, and applies offset only after merging. + auto params = make_float32_vector_search_params({0, 0, 0}, 20, 0); + auto& request = params.lance_scan_params.external_search_request; + request.__set_schema_version(1); + request.vector_search_options.__set_use_index(split < 2); + request.vector_search_options.__set_nprobes(4); + request.vector_search_options.__set_refine_factor(64); + auto& search = request.search_query.vector_search; + search.__set_column("vectors"); + search.__set_metric(TVectorMetric::COSINE); + search.query_vector.__set_dimension(128); + search.query_vector.__set_num_vectors(2); + std::string bytes(query.size() * sizeof(float), '\0'); + for (size_t j = 0; j < query.size(); ++j) { + LittleEndian::Store32(bytes.data() + j * sizeof(float), + std::bit_cast(query[j])); + } + search.query_vector.__set_values(bytes); + if (filtered) { + TSearchFilter filter; + filter.__set_format(TSearchFilterFormat::SQL); + filter.__set_payload("row_id % 5 = 0"); + request.__set_search_filter(filter); + request.__set_prefilter(true); + } + auto range = + make_lance_range(uri, fixture.version, {fixture.fragment_ids[split]}); + if (split < 2) { + range.table_format_params.lance_params.__set_index_segment_uuids( + {segments[split]}); + } + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + auto rows = read_vector_search_rows(&reader, &block); + ASSERT_EQ(20, rows.size()); + for (const auto& row : rows) EXPECT_EQ(split, (row.first - 1) % 3); + candidates.insert(candidates.end(), rows.begin(), rows.end()); + EXPECT_TRUE(reader.close().ok()); + } + std::sort(candidates.begin(), candidates.end(), order); + std::set unique; + for (const auto& row : candidates) ASSERT_TRUE(unique.insert(row.first).second); + for (int offset : {0, 7}) { + for (int j = 0; j < 13; ++j) { + EXPECT_EQ(oracle[offset + j].first, candidates[offset + j].first); + EXPECT_NEAR(oracle[offset + j].second, candidates[offset + j].second, 2e-5); + } + } + } + } + } +} + +TEST(LanceTableReaderVectorSearchTest, SplitWithoutIndexSegmentsSearchesFlat) { + // FE plans a split without index segments as a flat search. Lance must not search it with + // an index of its own choosing, even one covering the split and with use_index=true. + const std::filesystem::path uri = + "./be/test/format_v2/table/lance/data/multivector_ivf_flat.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + ASSERT_EQ(3, fixture.fragment_ids.size()); + std::vector segments; + ASSERT_TRUE(get_index_segment_uuids(uri, "vectors_idx", &segments).ok()); + ASSERT_EQ(2, segments.size()); + const auto query = representative_multi_vector(7, 0); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + std::map>> results; + for (const auto* mode : {"segment", "no_segment_default", "no_segment_use_index"}) { + SCOPED_TRACE(mode); + RuntimeProfile profile("lance_segmentless_split"); + auto params = make_float32_vector_search_params({0, 0, 0}, 5, 0); + auto& request = params.lance_scan_params.external_search_request; + request.vector_search_options.__isset.use_index = false; + if (std::string(mode) == "no_segment_use_index") { + request.vector_search_options.__set_use_index(true); + } + request.vector_search_options.__set_nprobes(64); + auto& search = request.search_query.vector_search; + search.__set_column("vectors"); + search.__set_metric(TVectorMetric::COSINE); + search.query_vector.__set_dimension(128); + search.query_vector.__set_num_vectors(1); + std::string bytes(query.size() * sizeof(float), '\0'); + for (size_t j = 0; j < query.size(); ++j) { + LittleEndian::Store32(bytes.data() + j * sizeof(float), + std::bit_cast(query[j])); + } + search.query_vector.__set_values(bytes); + auto range = make_lance_range(uri, fixture.version, {fixture.fragment_ids[0]}); + if (std::string(mode) == "segment") { + range.table_format_params.lance_params.__set_index_segment_uuids({segments[0]}); + } + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + results[mode] = read_vector_search_rows(&reader, &block); + ASSERT_EQ(5, results[mode].size()); + EXPECT_TRUE(reader.close().ok()); + auto* partitions = profile.get_counter("LanceIndexPartitionsSearched"); + ASSERT_NE(nullptr, partitions); + auto* flat = profile.get_counter("LancePlannedFlatSearchFragmentCount"); + ASSERT_NE(nullptr, flat); + if (std::string(mode) == "segment") { + EXPECT_GT(partitions->value(), 0); + EXPECT_EQ(0, flat->value()); + } else { + EXPECT_EQ(0, partitions->value()); + EXPECT_EQ(1, flat->value()); + } + } + EXPECT_EQ(results["no_segment_default"], results["no_segment_use_index"]); +} + +TEST(LanceTableReaderVectorSearchTest, RejectsIndexSegmentsWhenUseIndexIsFalse) { + // FE plans index segments only when use_index allows them. A split carrying segments under + // use_index=false is rejected instead of being searched with or without the index. + const std::filesystem::path uri = + "./be/test/format_v2/table/lance/data/multivector_ivf_flat.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + std::vector segments; + ASSERT_TRUE(get_index_segment_uuids(uri, "vectors_idx", &segments).ok()); + ASSERT_FALSE(segments.empty()); + const auto query = representative_multi_vector(7, 0); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (const bool with_segment : {true, false}) { + SCOPED_TRACE(with_segment ? "with segment" : "without segment"); + RuntimeProfile profile("lance_segment_with_use_index_false"); + auto params = make_float32_vector_search_params({0, 0, 0}, 5, 0); + auto& request = params.lance_scan_params.external_search_request; + ASSERT_TRUE(request.vector_search_options.__isset.use_index); + ASSERT_FALSE(request.vector_search_options.use_index); + auto& search = request.search_query.vector_search; + search.__set_column("vectors"); + search.__set_metric(TVectorMetric::COSINE); + search.query_vector.__set_dimension(128); + search.query_vector.__set_num_vectors(1); + std::string bytes(query.size() * sizeof(float), '\0'); + for (size_t j = 0; j < query.size(); ++j) { + LittleEndian::Store32(bytes.data() + j * sizeof(float), + std::bit_cast(query[j])); + } + search.query_vector.__set_values(bytes); + auto range = make_lance_range(uri, fixture.version, {fixture.fragment_ids[0]}); + if (with_segment) { + range.table_format_params.lance_params.__set_index_segment_uuids({segments[0]}); + } + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + const auto status = prepare_range(&reader, range); + if (with_segment) { + EXPECT_FALSE(status.ok()); + EXPECT_NE( + status.to_string().find("carries 1 index segments although use_index is false"), + std::string::npos) + << status.to_string(); + } else { + ASSERT_TRUE(status.ok()) << status.to_string(); + Block block; + add_output_columns(&block, columns); + EXPECT_EQ(5, read_vector_search_rows(&reader, &block).size()); + auto* flat = profile.get_counter("LancePlannedFlatSearchFragmentCount"); + ASSERT_NE(nullptr, flat); + EXPECT_EQ(1, flat->value()); + } + EXPECT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderVectorSearchTest, MultiVectorTopOnePreservesPrecisionAndBatchIndependence) { + const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/multivector.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (const int batch_size : {1, 2, 1024}) { + TQueryOptions options; + options.__set_batch_size(batch_size); + state.set_query_options(options); + for (const bool indexed : {false, true}) { + SCOPED_TRACE("batch_size=" + std::to_string(batch_size) + + " indexed=" + std::to_string(indexed)); + auto params = make_float32_vector_search_params({0, 0, 0}, 1, 0); + auto& request = params.lance_scan_params.external_search_request; + request.__set_schema_version(1); + request.vector_search_options.__set_use_index(indexed); + request.vector_search_options.__set_nprobes(1); + auto& search = request.search_query.vector_search; + search.__set_column(indexed ? "batch_vectors" : "tiny_vectors"); + search.__set_metric(indexed ? TVectorMetric::COSINE : TVectorMetric::L2); + auto& query = search.query_vector; + query.__set_dimension(2); + query.__set_num_vectors(indexed ? 2 : 1); + std::string values(indexed ? 16 : 8, '\0'); + if (indexed) { + LittleEndian::Store32(values.data(), std::bit_cast(1.0F)); + LittleEndian::Store32(values.data() + 12, std::bit_cast(1.0F)); + } + query.__set_values(values); + RuntimeProfile profile("multi_vector_top_one"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_range(&reader, indexed ? make_indexed_lance_range( + uri, fixture, fixture.fragment_ids, + "batch_vectors_idx") + : make_lance_range(uri, fixture.version, + fixture.fragment_ids)) + .ok()); + Block block; + add_output_columns(&block, columns); + auto rows = read_vector_search_rows(&reader, &block); + ASSERT_EQ(1, rows.size()); + EXPECT_EQ(indexed ? 3 : 2, rows[0].first); + EXPECT_NEAR(indexed ? 2.0F - std::sqrt(2.0F) : 1e-8F, rows[0].second, + indexed ? 1e-6F : 1e-13F); + EXPECT_TRUE(reader.close().ok()); + auto* partitions = profile.get_counter("LanceIndexPartitionsSearched"); + ASSERT_NE(nullptr, partitions); + EXPECT_EQ(indexed, partitions->value() > 0); + } + } +} + +TEST(LanceTableReaderVectorSearchTest, MultiVectorCosineMasksUndefinedRows) { + const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/multivector_zero.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (const bool indexed : {false, true}) { + for (const bool zero_query : {false, true}) { + auto params = make_float32_vector_search_params({0, 0, 0}, 10, 0); + auto& request = params.lance_scan_params.external_search_request; + request.__set_schema_version(1); + request.vector_search_options.__set_use_index(indexed); + request.vector_search_options.__set_nprobes(1); + auto& search = request.search_query.vector_search; + search.__set_column("vectors"); + search.__set_metric(TVectorMetric::COSINE); + auto& query = search.query_vector; + query.__set_dimension(2); + query.__set_num_vectors(2); + std::string values(16, '\0'); + if (!zero_query) { + LittleEndian::Store32(values.data(), std::bit_cast(1.0F)); + } + LittleEndian::Store32(values.data() + 12, std::bit_cast(1.0F)); + query.__set_values(values); + RuntimeProfile profile("multi_vector_cosine_zero_norm"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_range(&reader, indexed ? make_indexed_lance_range( + uri, fixture, fixture.fragment_ids, + "vectors_idx") + : make_lance_range(uri, fixture.version, + fixture.fragment_ids)) + .ok()); + Block block; + add_output_columns(&block, columns); + auto rows = read_vector_search_rows(&reader, &block); + std::sort(rows.begin(), rows.end()); + // Undefined pairs cannot poison valid matches in the same row or other fragments. + const std::vector> expected = + zero_query ? std::vector> {} + : std::vector> {{2, 1}, {3, 1}}; + EXPECT_EQ(expected, rows); + EXPECT_TRUE(reader.close().ok()); + auto* partitions = profile.get_counter("LanceIndexPartitionsSearched"); + ASSERT_NE(nullptr, partitions); + EXPECT_EQ(indexed, partitions->value() > 0); + } + } +} + +// FE plans an omitted metric as L2, and this column's only index is cosine, so FE plans every split +// flat. BE must then search flat too, with the same L2 metric in every split, although the request +// allows indexes. +TEST(LanceTableReaderVectorSearchTest, MultiVectorDefaultMetricIsConsistentAcrossFragments) { + const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/multivector.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + std::vector> all_rows; + for (const auto fragment : fixture.fragment_ids) { + auto params = make_float32_vector_search_params({0, 0, 0}, 10, 0); + auto& request = params.lance_scan_params.external_search_request; + request.__set_schema_version(1); + request.vector_search_options.__set_use_index(true); + auto& search = request.search_query.vector_search; + search.__set_column("vectors32"); + search.__isset.metric = false; + auto& query = search.query_vector; + query.__set_dimension(2); + query.__set_num_vectors(1); + std::string values(8, '\0'); + LittleEndian::Store32(values.data(), std::bit_cast(1.0F)); + query.__set_values(values); + RuntimeProfile profile("multi_vector_default_metric"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, {fragment}).ok()); + Block block; + add_output_columns(&block, columns); + auto rows = read_vector_search_rows(&reader, &block); + all_rows.insert(all_rows.end(), rows.begin(), rows.end()); + EXPECT_TRUE(reader.close().ok()); + auto* partitions = profile.get_counter("LanceIndexPartitionsSearched"); + ASSERT_NE(nullptr, partitions); + EXPECT_EQ(0, partitions->value()); + } + std::sort(all_rows.begin(), all_rows.end()); + EXPECT_EQ((std::vector> {{1, 0}, {2, 1}, {3, 4}, {6, 1}}), all_rows); +} + +TEST(LanceTableReaderVectorSearchTest, MultiVectorRejectsActualNullAndNonFiniteElements) { + const std::filesystem::path uri = + "./be/test/format_v2/table/lance/data/multivector_invalid_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + const Columns columns {projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (const auto& [bits, type, width] : {std::tuple {16, TVectorElementType::FLOAT16, 2}, + std::tuple {32, TVectorElementType::FLOAT32, 4}, + std::tuple {64, TVectorElementType::FLOAT64, 8}}) { + for (const auto* prefix : {"null", "nan", "inf"}) { + const auto column = std::string(prefix) + std::to_string(bits); + SCOPED_TRACE(column); + auto params = make_float32_vector_search_params({0, 0, 0}, 1, 0); + auto& request = params.lance_scan_params.external_search_request; + request.__set_schema_version(1); + auto& search = request.search_query.vector_search; + search.__set_column(column); + search.query_vector.__set_dimension(3); + search.query_vector.__set_element_type(type); + search.query_vector.__set_num_vectors(1); + search.query_vector.__set_values(std::string(3 * width, '\0')); + RuntimeProfile profile("multi_vector_invalid_values"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok()); + Block block; + add_output_columns(&block, columns); + bool eos = false; + auto status = reader.get_block(&block, &eos); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("finite, non-null elements"), std::string::npos) + << status.to_string(); + EXPECT_TRUE(reader.close().ok()); + } + } +} + +TEST(LanceTableReaderVectorSearchTest, RejectsInvalidQueryParallelismBeforeOpeningDataset) { + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + RuntimeProfile profile("lance_invalid_query_parallelism"); + auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0); + params.lance_scan_params.external_search_request.vector_search_options.__set_query_parallelism( + -2); + const Columns columns {projected_column("_distance", TYPE_FLOAT, true)}; + LanceTableReader reader; + const auto status = init_reader(&reader, columns, &state, &profile, ¶ms); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("query_parallelism")); +} + +TEST(LanceTableReaderVectorSearchTest, ValidatesDistanceRangeBeforeDatasetAccess) { + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + const Columns columns {projected_column("_distance", TYPE_FLOAT, true)}; + for (int mode = 0; mode < 6; ++mode) { + SCOPED_TRACE(mode); + auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0); + auto& request = params.lance_scan_params.external_search_request; + auto& vector = request.search_query.vector_search; + vector.__set_distance_lower_bound(1.0); + vector.__set_distance_upper_bound(2.0); + if (mode == 0) vector.__set_distance_upper_bound(std::numeric_limits::infinity()); + if (mode == 1) vector.__set_distance_lower_bound(std::numeric_limits::quiet_NaN()); + if (mode == 2) vector.__set_distance_upper_bound(std::numeric_limits::max()); + if (mode == 3) vector.__set_distance_upper_bound(1.0); + if (mode == 4) vector.__set_distance_upper_bound(1.00000001); + if (mode == 5) vector.query_vector.__set_num_vectors(1); + RuntimeProfile profile("invalid_distance_range"); + LanceTableReader reader; + const auto status = init_reader(&reader, columns, &state, &profile, ¶ms); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, + status.to_string().find(mode == 5 ? "multi-vector" : "distance")); + } +} + +TEST(LanceTableReaderVectorSearchTest, AppliesDistanceBoundsAndOffset) { + const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (int mode = 0; mode < 5; ++mode) { + SCOPED_TRACE(mode); + auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 10, mode == 4 ? 1 : 0); + auto& request = params.lance_scan_params.external_search_request; + auto& vector = request.search_query.vector_search; + if (mode != 1) vector.__set_distance_lower_bound(1.0); + if (mode != 2) vector.__set_distance_upper_bound(mode == 3 ? 1.1 : 8.25); + RuntimeProfile profile("distance_range"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok()); + Block block; + add_output_columns(&block, columns); + const auto rows = read_vector_search_rows(&reader, &block); + if (mode == 4) { + EXPECT_TRUE(rows.empty()); + } else { + ASSERT_FALSE(rows.empty()); + EXPECT_EQ(mode == 1 ? 1 : 2, rows[0].first); + for (const auto& [id, distance] : rows) { + if (mode != 1) EXPECT_GE(distance, 1.0F); + if (mode != 2) EXPECT_LT(distance, mode == 3 ? 1.1F : 8.25F); + } + if (mode == 0 || mode == 3) EXPECT_EQ(1U, rows.size()); + } + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderVectorSearchTest, DistanceRangeFallbackPreservesExactScores) { + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (const bool dot : {false, true}) { + const std::filesystem::path uri = + dot ? "./be/test/format_v2/table/lance/data/distance_range_dot.lance" + : "./be/test/format_v2/table/lance/data/distance_range_sq.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + for (const bool use_index : {false, true}) { + for (const bool filtered : {false, true}) { + SCOPED_TRACE(std::to_string(dot) + "/" + std::to_string(use_index) + "/" + + std::to_string(filtered)); + const float max = std::numeric_limits::max(); + auto params = make_float32_vector_search_params( + dot ? std::array {-max, 0.0F, 0.0F} + : std::array {0.51F, 0.0F, 0.0F}, + 300, 0, + filtered ? std::optional("row_id >= 0") : std::nullopt); + auto& request = params.lance_scan_params.external_search_request; + auto& vector = request.search_query.vector_search; + vector.__set_distance_lower_bound(dot ? max : 0.0001); + if (dot) { + vector.__set_metric(TVectorMetric::DOT_PRODUCT); + } else { + vector.__set_distance_upper_bound(0.001); + } + request.vector_search_options.__set_use_index(use_index); + request.vector_search_options.__set_refine_factor(1); + request.vector_search_options.__set_query_parallelism(4); + // Range fallback splits have no index UUIDs even when use_index was requested. + // SQ quantization must not drop 0.0004, nor an absent upper bound drop FLT_MAX. + RuntimeProfile profile("distance_range_fallback"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok()); + Block block; + add_output_columns(&block, columns); + const auto rows = read_vector_search_rows(&reader, &block); + ASSERT_EQ(1U, rows.size()); + EXPECT_EQ(dot ? 0 : 256, rows[0].first); + if (dot) { + EXPECT_FLOAT_EQ(max, rows[0].second); + } else { + EXPECT_NEAR(0.0004F, rows[0].second, 1e-8F); + } + ASSERT_TRUE(reader.close().ok()); + } + } + } +} + +TEST(LanceTableReaderVectorSearchTest, DistanceRangeCoversIndexedAndAppendedFragments) { + const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/distance_range.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + ASSERT_EQ(2U, fixture.fragment_ids.size()); + std::vector segments; + ASSERT_TRUE(get_index_segment_uuids(uri, "embedding_idx", &segments).ok()); + ASSERT_EQ(1U, segments.size()); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (const bool use_index : {false, true}) { + for (const bool filtered : {false, true}) { + std::vector ids; + for (size_t split = 0; split < fixture.fragment_ids.size(); ++split) { + SCOPED_TRACE(std::to_string(use_index) + "/" + std::to_string(filtered) + "/" + + std::to_string(split)); + auto params = make_float32_vector_search_params( + {127.0F, 0.0F, 0.0F}, 10, 0, + filtered ? std::optional("row_id >= 127") : std::nullopt); + auto& request = params.lance_scan_params.external_search_request; + request.search_query.vector_search.__set_distance_lower_bound(1.0); + request.search_query.vector_search.__set_distance_upper_bound(4.0); + const bool indexed_split = use_index && split == 0; + request.vector_search_options.__set_use_index(indexed_split); + request.vector_search_options.__set_nprobes(1); + auto range = make_lance_range(uri, fixture.version, {fixture.fragment_ids[split]}); + if (indexed_split) { + range.table_format_params.lance_params.__set_index_segment_uuids(segments); + } + RuntimeProfile profile("mixed_distance_range"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + for (const auto& [id, distance] : read_vector_search_rows(&reader, &block)) { + ids.push_back(id); + EXPECT_FLOAT_EQ(1.0F, distance); + } + ASSERT_TRUE(reader.close().ok()); + } + std::ranges::sort(ids); + EXPECT_EQ(filtered ? std::vector {128} : std::vector({126, 128}), + ids); + } + } +} + +TEST(LanceTableReaderVectorSearchTest, QueryParallelismPreservesResults) { + const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(uri, &fixture).ok()); + const Columns columns {projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true)}; + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + for (const int parallelism : {-1, 0, 1, 4}) { + SCOPED_TRACE(parallelism); + auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 1); + params.lance_scan_params.external_search_request.vector_search_options + .__set_query_parallelism(parallelism); + RuntimeProfile profile("lance_query_parallelism"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, ¶ms).ok()); + ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok()); + Block block; + add_output_columns(&block, columns); + const auto rows = read_vector_search_rows(&reader, &block); + ASSERT_EQ(2U, rows.size()); + EXPECT_EQ(2, rows[0].first); + EXPECT_EQ(4, rows[1].first); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderVectorSearchTest, SearchesWholeSnapshotWithOffsetAndDistance) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true), + }; + TQueryOptions query_options; + query_options.__set_batch_size(1); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_vector_search_fixture"); + auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 1); + auto& search_options = + scan_params.lance_scan_params.external_search_request.vector_search_options; + search_options.__set_nprobes(4); + search_options.__set_refine_factor(2); + search_options.__set_ef(16); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + const auto rows = read_vector_search_rows(&reader, &block); + ASSERT_EQ(2U, rows.size()); + EXPECT_EQ(2, rows[0].first); + EXPECT_FLOAT_EQ(1.0F, rows[0].second); + EXPECT_EQ(4, rows[1].first); + EXPECT_FLOAT_EQ(8.25F, rows[1].second); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderVectorSearchTest, AppliesSearchFilterBeforeTopKWhenPrefilterEnabled) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true), + }; + TQueryOptions query_options; + query_options.__set_batch_size(2); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_vector_search_prefilter_fixture"); + auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 1, 0, "row_id >= 3"); + scan_params.lance_scan_params.external_search_request.__set_prefilter(true); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + const auto rows = read_vector_search_rows(&reader, &block); + ASSERT_EQ(1U, rows.size()); + EXPECT_EQ(4, rows[0].first); + EXPECT_FLOAT_EQ(8.25F, rows[0].second); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderVectorSearchTest, SearchFilterDefaultsToPostfilter) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true), + }; + TQueryOptions query_options; + query_options.__set_batch_size(2); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_vector_search_postfilter_default_fixture"); + auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 1, 0, "row_id >= 3"); + const auto& request = scan_params.lance_scan_params.external_search_request; + ASSERT_FALSE(request.__isset.prefilter); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + const auto rows = read_vector_search_rows(&reader, &block); + // The nearest row has row_id < 3, so filtering the top-1 candidate leaves nothing. With + // prefilter enabled, the same request returns row 4 (see the test above). + EXPECT_TRUE(rows.empty()); + EXPECT_TRUE(reader.close().ok()); + + for (const char* name : + {"RowIdPrefilterLoads", "RowIdPrefilterInputRows", "RowIdPrefilterInputBatches", + "RowIdPrefilterIds", "RowIdPrefilterLoadTime", "RowIdPrefilterInputTime", + "RowIdPrefilterBuildTime"}) { + auto* counter = profile.get_counter(name); + ASSERT_NE(nullptr, counter) << name; + EXPECT_EQ(0, counter->value()) << name; + } +} + +TEST(LanceTableReaderVectorSearchTest, SearchesMultipleFragmentSplits) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_GT(fixture.fragment_ids.size(), 1U); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("_distance", TYPE_FLOAT, true), + }; + TQueryOptions query_options; + query_options.__set_batch_size(2); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_vector_search_fragment_splits_fixture"); + auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 4, 0); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + std::vector row_ids; + for (const auto fragment_id : fixture.fragment_ids) { + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, {fragment_id}).ok()); + Block block; + add_output_columns(&block, columns); + const auto rows = read_vector_search_rows(&reader, &block); + for (const auto& row : rows) { + row_ids.emplace_back(row.first); + } + } + std::ranges::sort(row_ids); + EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderVectorSearchTest, ReturnsStableGlobalRowIdsAndFetchesPayload) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const auto global_rowid_name = BeConsts::GLOBAL_ROWID_COL + std::string("topn_fetch_lance"); + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column(global_rowid_name, TYPE_STRING, false), + }; + TQueryOptions query_options; + query_options.__set_batch_size(1); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_vector_search_global_rowid_fixture"); + auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 4, 0); + const GlobalRowIdContext context {.backend_id = 123456789, .file_id = 42}; + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + const auto scan_row_ids = [&]() { + std::map row_ids; + const auto prepare_status = + prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids, context); + EXPECT_TRUE(prepare_status.ok()) << prepare_status.to_string(); + if (!prepare_status.ok()) { + return row_ids; + } + Block block; + add_output_columns(&block, columns); + bool eos = false; + while (!eos) { + const auto read_status = reader.get_block(&block, &eos); + EXPECT_TRUE(read_status.ok()) << read_status.to_string(); + if (!read_status.ok()) { + break; + } + if (eos) { + continue; + } + const auto& logical_row_ids = + assert_cast(*block.get_by_position(0).column); + const auto& global_row_ids = + assert_cast(*block.get_by_position(1).column); + for (size_t row = 0; row < block.rows(); ++row) { + const auto location = decode_lance_row_id(global_row_ids, row); + EXPECT_EQ(static_cast(ROW_VERSION::LANCE_DATASET_ROW_ID), + location.version); + EXPECT_EQ(context.backend_id, location.backend_id); + EXPECT_EQ(context.file_id, location.lance_file_id); + row_ids.emplace(logical_row_ids.get_data()[row], location.lance_row_id); + } + } + return row_ids; + }; + + const auto first_scan = scan_row_ids(); + const auto second_scan = scan_row_ids(); + EXPECT_EQ(4U, first_scan.size()); + EXPECT_EQ(first_scan, second_scan); + EXPECT_TRUE(reader.close().ok()); + + ASSERT_TRUE(first_scan.contains(2)); + ASSERT_TRUE(first_scan.contains(4)); + const std::vector fetch_row_ids {first_scan.at(4), first_scan.at(2), + first_scan.at(4)}; + const Columns payload_columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("label", TYPE_STRING, true), + }; + RuntimeProfile fetch_profile("lance_vector_search_rowid_fetch_fixture"); + LanceTableReader payload_reader; + ASSERT_TRUE(init_reader(&payload_reader, payload_columns, &state, &fetch_profile, &scan_params) + .ok()); + Block payload_block; + add_output_columns(&payload_block, payload_columns); + ASSERT_TRUE(payload_reader + .read_by_row_ids(make_lance_range(dataset_uri, fixture.version, + fixture.fragment_ids), + fetch_row_ids, &payload_block) + .ok()); + + ASSERT_EQ(3U, payload_block.rows()); + const auto& logical_row_ids = + assert_cast(*payload_block.get_by_position(0).column); + const auto& labels = + assert_cast(*payload_block.get_by_position(1).column); + const auto& label_values = assert_cast(labels.get_nested_column()); + EXPECT_EQ((std::vector {4, 2, 4}), + std::vector(logical_row_ids.get_data().begin(), + logical_row_ids.get_data().end())); + ASSERT_EQ(3U, labels.size()); + EXPECT_EQ(0, labels.get_null_map_data()[0]); + EXPECT_EQ(0, labels.get_null_map_data()[1]); + EXPECT_EQ(0, labels.get_null_map_data()[2]); + EXPECT_EQ("extra", label_values.get_data_at(0).to_string()); + EXPECT_EQ("unit-x", label_values.get_data_at(1).to_string()); + EXPECT_EQ("extra", label_values.get_data_at(2).to_string()); + ASSERT_NE(nullptr, fetch_profile.get_counter("LanceRowIdFetchRows")); + EXPECT_EQ(3, fetch_profile.get_counter("LanceRowIdFetchRows")->value()); + ASSERT_NE(nullptr, fetch_profile.get_counter("LanceRowIdFetchCalls")); + EXPECT_EQ(1, fetch_profile.get_counter("LanceRowIdFetchCalls")->value()); + Block second_block; + add_output_columns(&second_block, payload_columns); + ASSERT_TRUE(payload_reader + .read_by_row_ids(make_lance_range(dataset_uri, fixture.version, + fixture.fragment_ids), + fetch_row_ids, &second_block) + .ok()); + EXPECT_EQ(6, fetch_profile.get_counter("LanceRowIdFetchRows")->value()); + EXPECT_EQ(2, fetch_profile.get_counter("LanceRowIdFetchCalls")->value()); + ASSERT_TRUE(payload_reader + .read_by_row_ids(make_lance_range(dataset_uri, fixture.version, + fixture.fragment_ids), + {}, &second_block) + .ok()); + EXPECT_EQ(2, fetch_profile.get_counter("LanceRowIdFetchCalls")->value()); + EXPECT_TRUE(payload_reader.close().ok()); +} + +TEST(LanceTableReaderVectorSearchTest, ReadsOnlyGlobalRowIdVirtualColumn) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const auto global_rowid_name = BeConsts::GLOBAL_ROWID_COL + std::string("topn_fetch_lance"); + const Columns columns {projected_column(global_rowid_name, TYPE_STRING, false)}; + TQueryOptions query_options; + query_options.__set_batch_size(2); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_vector_search_only_global_rowid_fixture"); + auto scan_params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0); + const GlobalRowIdContext context {.backend_id = 13579, .file_id = 24}; + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids, context).ok()); + + Block block; + add_output_columns(&block, columns); + std::set native_row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& global_row_ids = + assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < block.rows(); ++row) { + const auto location = decode_lance_row_id(global_row_ids, row); + EXPECT_EQ(static_cast(ROW_VERSION::LANCE_DATASET_ROW_ID), location.version); + EXPECT_EQ(context.backend_id, location.backend_id); + EXPECT_EQ(context.file_id, location.lance_file_id); + native_row_ids.emplace(location.lance_row_id); + } + } + EXPECT_EQ(2U, native_row_ids.size()); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderFilterTest, CombinesStaticSubstraitFilterWithRuntimeFilter) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns {projected_column("label", TYPE_STRING, true)}; + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_filter_pushdown_fixture"); + TFileScanRangeParams scan_params; + // Generated by the FE LancePredicateConverter for row_id >= 3 against this + // fixture's complete Arrow schema. Keeping the FE-produced envelope here + // verifies Java Substrait -> thrift binary -> lance-c interoperability. + const std::string substrait_filter_base64 = + "Ch4IARIaL2Z1bmN0aW9uc19jb21wYXJpc29uLnlhbWwSERoPCAEaC2d0ZTphbnlfYW55GisKHBoaGgQKAhACIg" + "oa" + "CBIGCgISACIAIgYaBAoCOAMaC2ZpbHRlcl9tYXNrIrkGCgZyb3dfaWQKCmJvb2xfdmFsdWUKDXRpbnlpbnRfdm" + "Fs" + "dWUKDnNtYWxsaW50X3ZhbHVlCglpbnRfdmFsdWUKDGJpZ2ludF92YWx1ZQoLZmxvYXRfdmFsdWUKDGRvdWJsZV" + "92" + "YWx1ZQoLZGVjaW1hbF8xXzAKC2RlY2ltYWxfOV8yCgxkZWNpbWFsXzEwXzAKDGRlY2ltYWxfMThfNAoMZGVjaW" + "1h" + "bF8xOV8wCg1kZWNpbWFsXzM4XzEwCiRfX3VubGlrZWx5X25hbWVfcGxhY2Vob2xkZXJfZG9yaXNfMTQKJF9fdW" + "5s" + "aWtlbHlfbmFtZV9wbGFjZWhvbGRlcl9kb3Jpc18xNQoKdGV4dF92YWx1ZQokX191bmxpa2VseV9uYW1lX3BsYW" + "Nl" + "aG9sZGVyX2RvcmlzXzE3CiRfX3VubGlrZWx5X25hbWVfcGxhY2Vob2xkZXJfZG9yaXNfMTgKJF9fdW5saWtlbH" + "lf" + "bmFtZV9wbGFjZWhvbGRlcl9kb3Jpc18xOQokX191bmxpa2VseV9uYW1lX3BsYWNlaG9sZGVyX2RvcmlzXzIwCi" + "Rf" + "X3VubGlrZWx5X25hbWVfcGxhY2Vob2xkZXJfZG9yaXNfMjEKJF9fdW5saWtlbHlfbmFtZV9wbGFjZWhvbGRlcl" + "9k" + "b3Jpc18yMgokX191bmxpa2VseV9uYW1lX3BsYWNlaG9sZGVyX2RvcmlzXzIzCiRfX3VubGlrZWx5X25hbWVfcG" + "xh" + "Y2Vob2xkZXJfZG9yaXNfMjQKBWxhYmVsCiRfX3VubGlrZWx5X25hbWVfcGxhY2Vob2xkZXJfZG9yaXNfMjYSxw" + "EK" + "BDoCEAIKBAoCEAEKBBICEAEKBBoCEAEKBCoCEAEKBDoCEAEKBFICEAEKBFoCEAEKB8IBBBABIAEKCcIBBggCEA" + "kg" + "AQoHwgEEEAogAQoJwgEGCAQQEiABCgfCAQQQEyABCgnCAQYIChAmIAEKBfIBAhgBCgXyAQIYAQoEYgIQAQoF8g" + "EC" + "GAEKBfIBAhgBCgXyAQIYAQoF8gECGAEKBfIBAhgBCgXyAQIYAQoF8gECGAEKBfIBAhgBCgRiAhABCgXyAQIYAR" + "gC" + "Og8QRioLZG9yaXMtbGFuY2U="; + std::string substrait_filter; + ASSERT_TRUE(base64_decode(substrait_filter_base64, &substrait_filter)); + TLanceScanParams lance_scan_params; + lance_scan_params.__set_lance_substrait_filter(std::move(substrait_filter)); + scan_params.__set_lance_scan_params(std::move(lance_scan_params)); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + ASSERT_TRUE(reader.TEST_conjuncts_empty()); + + Block block; + add_output_columns(&block, columns); + std::vector labels; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& nullable = assert_cast(*block.get_by_position(0).column); + const auto& values = assert_cast(nullable.get_nested_column()); + for (size_t row = 0; row < block.rows(); ++row) { + ASSERT_EQ(0, nullable.get_null_map_data()[row]); + labels.emplace_back(values.get_data_at(row).to_string()); + } + } + std::ranges::sort(labels); + EXPECT_EQ((std::vector {"extra", "mixed"}), labels); + EXPECT_TRUE(reader.close().ok()); + + // The static Substrait prefilter and a later runtime filter must both remain active. Static + // row_id >= 3 yields {3, 4}; runtime row_id IN (2, 4) yields {2, 4}; their intersection is {4}. + std::string combined_substrait_filter; + ASSERT_TRUE(base64_decode(substrait_filter_base64, &combined_substrait_filter)); + TLanceScanParams combined_lance_scan_params; + combined_lance_scan_params.__set_lance_substrait_filter(std::move(combined_substrait_filter)); + TFileScanRangeParams combined_scan_params; + combined_scan_params.__set_lance_scan_params(std::move(combined_lance_scan_params)); + const Columns row_id_columns {projected_column("row_id", TYPE_BIGINT, false)}; + RuntimeProfile combined_profile("lance_substrait_and_runtime_filter_fixture"); + const auto combined_runtime_filter = create_int64_runtime_in_conjunct("row_id", {2, 4}, 42); + + LanceTableReader combined_reader; + ASSERT_TRUE(init_reader(&combined_reader, row_id_columns, &state, &combined_profile, + &combined_scan_params, {combined_runtime_filter}) + .ok()); + ASSERT_TRUE(prepare_fixture(&combined_reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block combined_block; + add_output_columns(&combined_block, row_id_columns); + std::vector combined_row_ids; + eos = false; + while (!eos) { + ASSERT_TRUE(combined_reader.get_block(&combined_block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = + assert_cast(*combined_block.get_by_position(0).column); + combined_row_ids.insert(combined_row_ids.end(), row_ids.get_data().begin(), + row_ids.get_data().end()); + } + EXPECT_EQ((std::vector {4}), combined_row_ids); + EXPECT_TRUE(combined_reader.close().ok()); +} + +TEST(LanceTableReaderScalarSegmentTest, FiltersIndexedAndUncoveredDomainsWithoutLosingRows) { + const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + for (const auto index_type : {LANCE_SCALAR_BTREE, LANCE_SCALAR_BITMAP}) { + SCOPED_TRACE(index_type == LANCE_SCALAR_BTREE ? "BTREE" : "BITMAP"); + const auto dataset_uri = std::filesystem::temp_directory_path() / + ("doris_lance_scalar_segment_" + std::to_string(unique_suffix) + + "_" + std::to_string(index_type) + ".lance"); + Defer cleanup {[&] { + std::error_code error; + std::filesystem::remove_all(dataset_uri, error); + }}; + const auto schema = arrow::schema({arrow::field("row_id", arrow::int64(), false)}); + for (int64_t first_id : {1, 3}) { + arrow::Int64Builder values; + ASSERT_TRUE(values.AppendValues({first_id, first_id + 1}).ok()); + auto array = values.Finish(); + ASSERT_TRUE(array.ok()); + auto batch = arrow::RecordBatch::Make(schema, 2, {std::move(array).ValueUnsafe()}); + auto batches = arrow::RecordBatchReader::Make({batch}, schema); + ASSERT_TRUE(batches.ok()); + ArrowArrayStream stream {}; + ASSERT_TRUE( + arrow::ExportRecordBatchReader(std::move(batches).ValueUnsafe(), &stream).ok()); + auto dataset = ::lance::Dataset::write( + dataset_uri.string(), &stream, + first_id == 1 ? ::lance::WriteMode::Create : ::lance::WriteMode::Append); + if (first_id == 1) { + // The second append remains uncovered by this index segment. + dataset.create_scalar_index("row_id", index_type, "row_id_idx"); + } + } + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_EQ(2, fixture.fragment_ids.size()); + std::vector segments; + ASSERT_TRUE(get_index_segment_uuids(dataset_uri, "row_id_idx", &segments).ok()); + ASSERT_EQ(1, segments.size()); + + const auto read_domain = [&](const std::vector& fragments, bool use_segment, + const std::vector& expected) { + SCOPED_TRACE(::testing::PrintToString(fragments)); + SCOPED_TRACE(use_segment ? "selected segment" : "scalar index disabled"); + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + RuntimeProfile profile("lance_scalar_segment"); + TFileScanRangeParams scan_params; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + LanceTableReader reader; + auto filter = create_int64_runtime_in_conjunct("row_id", {2, 4}, 41); + ASSERT_TRUE( + init_reader(&reader, columns, &state, &profile, &scan_params, {filter}).ok()); + auto range = make_lance_range(dataset_uri, fixture.version, fragments); + if (use_segment) { + range.table_format_params.lance_params.__set_index_segment_uuids(segments); + } else { + range.table_format_params.lance_params.__set_use_scalar_index(false); + } + ASSERT_TRUE(prepare_range(&reader, std::move(range)).ok()); + Block block; + add_output_columns(&block, columns); + std::vector actual; + bool eos = false; + while (!eos) { + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (!eos) { + const auto& ids = + assert_cast(*block.get_by_position(0).column); + actual.insert(actual.end(), ids.get_data().begin(), ids.get_data().end()); + } + } + std::ranges::sort(actual); + EXPECT_EQ(expected, actual); + EXPECT_TRUE(reader.close().ok()); + }; + read_domain({fixture.fragment_ids[0]}, true, {2}); + read_domain({fixture.fragment_ids[1]}, false, {4}); + // A segment which cannot cover the whole task must filter the entire explicit domain. + read_domain(fixture.fragment_ids, true, {2, 4}); + // Disabling the index must produce the same rows in each read domain. + read_domain({fixture.fragment_ids[0]}, false, {2}); + read_domain(fixture.fragment_ids, false, {2, 4}); + } +} + +TEST(LanceTableReaderScalarSegmentTest, RejectsInvalidSegmentAssignments) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + const auto check_invalid = [&](TLanceFileDesc params, const std::string& message) { + TQueryGlobals globals; + RuntimeState state(TQueryOptions(), globals); + RuntimeProfile profile("lance_invalid_scalar_segment"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, {projected_column("row_id", TYPE_BIGINT, false)}, &state, + &profile, &scan_params) + .ok()); + auto range = make_lance_range(dataset_uri, fixture.version, fixture.fragment_ids); + range.table_format_params.__set_lance_params(std::move(params)); + auto status = prepare_range(&reader, std::move(range)); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find(message), std::string::npos) << status.to_string(); + EXPECT_TRUE(reader.close().ok()); + }; + auto params = make_lance_range(dataset_uri, fixture.version, fixture.fragment_ids) + .table_format_params.lance_params; + params.__set_index_segment_uuids({"too-short"}); + check_invalid(params, "16 bytes"); + params.__set_index_segment_uuids({std::string(16, 'a'), std::string(16, 'b')}); + check_invalid(params, "only one scalar index segment"); + params.__set_index_segment_uuids({std::string(16, 'a')}); + params.__set_fragment_ids({}); + check_invalid(params, "nonempty fragment ids"); + params.__set_fragment_ids(fixture.fragment_ids); + params.__set_version(0); + check_invalid(params, "fixed version"); + params.__set_version(fixture.version); + params.__set_use_scalar_index(false); + check_invalid(params, "use_scalar_index=false"); +} + +TEST(LanceTableReaderFilterTest, PushesRuntimeInFilterIntoLanceScanner) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_runtime_filter_pushdown_fixture"); + TFileScanRangeParams scan_params; + const auto runtime_filter = create_int64_runtime_in_conjunct("row_id", {2, 4}, 41); + + LanceTableReader reader; + ASSERT_TRUE( + init_reader(&reader, columns, &state, &profile, &scan_params, {runtime_filter}).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + std::vector actual_row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + actual_row_ids.insert(actual_row_ids.end(), row_ids.get_data().begin(), + row_ids.get_data().end()); + } + std::ranges::sort(actual_row_ids); + EXPECT_EQ((std::vector {2, 4}), actual_row_ids); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderFilterTest, SkipsNullAwareRuntimeRangeBeforeLanceScanner) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_null_aware_runtime_filter_fixture"); + TFileScanRangeParams scan_params; + const VExprContextSPtrs runtime_filters { + create_runtime_range_conjunct("bigint_value", TExprOpcode::GE, + std::numeric_limits::lowest(), + std::make_shared(), 43, true), + create_runtime_range_conjunct("bigint_value", TExprOpcode::LE, + std::numeric_limits::max(), + std::make_shared(), 43, true), + }; + + LanceTableReader reader; + ASSERT_TRUE( + init_reader(&reader, columns, &state, &profile, &scan_params, runtime_filters).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = assert_cast(*block.get_by_position(0).column); + row_ids.insert(row_ids.end(), values.get_data().begin(), values.get_data().end()); + } + } + std::ranges::sort(row_ids); + EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderFilterTest, SkipsUnsafeStringRuntimeFiltersBeforeLanceCStringBoundary) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_unsafe_string_runtime_filter_fixture"); + TFileScanRangeParams scan_params; + const VExprContextSPtrs runtime_filters { + create_string_runtime_in_conjunct("text_value", std::string("a\0b", 3), 44), + create_string_runtime_in_conjunct("text_value", std::string(1, static_cast(0xff)), + 45), + }; + + LanceTableReader reader; + ASSERT_TRUE( + init_reader(&reader, columns, &state, &profile, &scan_params, runtime_filters).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + rows += block.rows(); + } + } + EXPECT_EQ(4U, rows); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderFilterTest, SkipsTimestampNanoRuntimeFilterBeforeMaterialization) { + const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + const auto dataset_uri = + std::filesystem::temp_directory_path() / + ("doris_lance_timestamp_nano_rf_" + std::to_string(unique_suffix) + ".lance"); + Defer cleanup {[&] { + std::error_code error; + std::filesystem::remove_all(dataset_uri, error); + }}; + + arrow::Int64Builder row_id_builder; + ASSERT_TRUE(row_id_builder.Append(1).ok()); + std::shared_ptr row_ids; + ASSERT_TRUE(row_id_builder.Finish(&row_ids).ok()); + const auto timestamp_type = arrow::timestamp(arrow::TimeUnit::NANO); + arrow::TimestampBuilder timestamp_builder(timestamp_type, arrow::default_memory_pool()); + ASSERT_TRUE(timestamp_builder.Append(123456789).ok()); + std::shared_ptr timestamps; + ASSERT_TRUE(timestamp_builder.Finish(×tamps).ok()); + const auto schema = arrow::schema({arrow::field("row_id", arrow::int64()), + arrow::field("timestamp_value", timestamp_type)}); + ASSERT_TRUE(write_lance_record_batch(dataset_uri, + arrow::RecordBatch::Make(schema, 1, {row_ids, timestamps})) + .ok()); + + DateV2Value bound; + bound.unchecked_set_time(1970, 1, 1, 0, 0, 0, 123456); + const auto runtime_filter = create_runtime_range_conjunct( + "timestamp_value", TExprOpcode::LE, bound, std::make_shared(6), 46); + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("timestamp_value", + make_nullable(std::make_shared(6))), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_timestamp_nano_runtime_filter_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE( + init_reader(&reader, columns, &state, &profile, &scan_params, {runtime_filter}).ok()); + ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok()); + + Block block; + add_output_columns(&block, columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(1U, block.rows()); + const auto& values = assert_cast(*block.get_by_position(0).column); + EXPECT_EQ(1, values.get_data()[0]); + const auto& timestamp = assert_cast(*block.get_by_position(1).column); + // The physical value has .123456789, while Doris materialization truncates it to .123456. + // Therefore the residual <= .123456 accepts this row and the pre-materialization SQL must not + // remove it. + EXPECT_EQ("1970-01-01 00:00:00.123456", columns[1].type->to_string(timestamp, 0)); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderFilterTest, PreservesNegativeNaNUnderFloatingMinRuntimeFilter) { + const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + const auto dataset_uri = + std::filesystem::temp_directory_path() / + ("doris_lance_negative_nan_rf_" + std::to_string(unique_suffix) + ".lance"); + Defer cleanup {[&] { + std::error_code error; + std::filesystem::remove_all(dataset_uri, error); + }}; + + // Rows 1..5 hold 0.5, 1, 2, +NaN, and a negative NaN in both columns. + const auto negative_nan_f32 = std::bit_cast(uint32_t {0xffc00000}); + const auto negative_nan_f64 = std::bit_cast(uint64_t {0xfff8000000000000}); + ASSERT_TRUE(std::isnan(negative_nan_f32) && std::signbit(negative_nan_f32)); + ASSERT_TRUE(std::isnan(negative_nan_f64) && std::signbit(negative_nan_f64)); + arrow::Int64Builder row_id_builder; + ASSERT_TRUE(row_id_builder.AppendValues({1, 2, 3, 4, 5}).ok()); + std::shared_ptr row_ids; + ASSERT_TRUE(row_id_builder.Finish(&row_ids).ok()); + arrow::FloatBuilder f32_builder; + ASSERT_TRUE(f32_builder + .AppendValues({0.5F, 1.0F, 2.0F, std::numeric_limits::quiet_NaN(), + negative_nan_f32}) + .ok()); + std::shared_ptr f32_values; + ASSERT_TRUE(f32_builder.Finish(&f32_values).ok()); + arrow::DoubleBuilder f64_builder; + ASSERT_TRUE(f64_builder + .AppendValues({0.5, 1.0, 2.0, std::numeric_limits::quiet_NaN(), + negative_nan_f64}) + .ok()); + std::shared_ptr f64_values; + ASSERT_TRUE(f64_builder.Finish(&f64_values).ok()); + const auto schema = arrow::schema({arrow::field("row_id", arrow::int64()), + arrow::field("f32", arrow::float32()), + arrow::field("f64", arrow::float64())}); + ASSERT_TRUE(write_lance_record_batch( + dataset_uri, + arrow::RecordBatch::Make(schema, 5, {row_ids, f32_values, f64_values})) + .ok()); + + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + for (const bool use_double : {false, true}) { + SCOPED_TRACE(use_double ? "f64" : "f32"); + const auto runtime_filter = use_double ? create_runtime_range_conjunct( + "f64", TExprOpcode::GE, 1.0, + std::make_shared(), 52) + : create_runtime_range_conjunct( + "f32", TExprOpcode::GE, 1.0F, + std::make_shared(), 51); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_negative_nan_runtime_filter_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params, {runtime_filter}) + .ok()); + ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok()); + + Block block; + add_output_columns(&block, columns); + std::vector returned_row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column); + returned_row_ids.insert(returned_row_ids.end(), values.get_data().begin(), + values.get_data().end()); + } + } + EXPECT_TRUE(reader.close().ok()); + const auto* skipped_ids = profile.get_info_string("LanceRuntimeFilterSkippedIds"); + ASSERT_NE(nullptr, skipped_ids); + EXPECT_EQ(use_double ? "52" : "51", *skipped_ids); + // Doris keeps rows 2..5 for `x >= 1`, both NaNs included. A pushed `x >= 1` let Lance drop + // row 5, which sorts below every finite value in totalOrder. Row 1 may come back too: the + // Doris residual removes it. + std::ranges::sort(returned_row_ids); + for (const int64_t row_id : {2, 3, 4, 5}) { + EXPECT_TRUE(std::ranges::binary_search(returned_row_ids, row_id)) << row_id; + } + } +} + +TEST(LanceTableReaderFilterTest, SkipsPhysicalNumericTypesUnsupportedByPinnedPlanner) { + const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + const auto dataset_uri = + std::filesystem::temp_directory_path() / + ("doris_lance_unsupported_numeric_rf_" + std::to_string(unique_suffix) + ".lance"); + Defer cleanup {[&] { + std::error_code error; + std::filesystem::remove_all(dataset_uri, error); + }}; + + arrow::Int64Builder row_id_builder; + ASSERT_TRUE(row_id_builder.Append(1).ok()); + std::shared_ptr row_ids; + ASSERT_TRUE(row_id_builder.Finish(&row_ids).ok()); + arrow::HalfFloatBuilder half_float_builder; + ASSERT_TRUE(half_float_builder.Append(0x3E00).ok()); + std::shared_ptr half_floats; + ASSERT_TRUE(half_float_builder.Finish(&half_floats).ok()); + const auto scaled_decimal_type = arrow::decimal128(9, 2); + arrow::Decimal128Builder scaled_decimal_builder(scaled_decimal_type, + arrow::default_memory_pool()); + ASSERT_TRUE(scaled_decimal_builder.Append(arrow::Decimal128(12345)).ok()); + std::shared_ptr scaled_decimals; + ASSERT_TRUE(scaled_decimal_builder.Finish(&scaled_decimals).ok()); + const auto wide_decimal_type = arrow::decimal128(38, 0); + arrow::Decimal128Builder wide_decimal_builder(wide_decimal_type, arrow::default_memory_pool()); + const uint64_t wide_value = static_cast(std::numeric_limits::max()) + 1; + ASSERT_TRUE(wide_decimal_builder.Append(arrow::Decimal128(0, wide_value)).ok()); + std::shared_ptr wide_decimals; + ASSERT_TRUE(wide_decimal_builder.Finish(&wide_decimals).ok()); + arrow::UInt64Builder uint64_builder; + ASSERT_TRUE(uint64_builder.Append(wide_value).ok()); + std::shared_ptr uint64_values; + ASSERT_TRUE(uint64_builder.Finish(&uint64_values).ok()); + const auto schema = arrow::schema({ + arrow::field("row_id", arrow::int64()), + arrow::field("half_float", arrow::float16()), + arrow::field("scaled_decimal", scaled_decimal_type), + arrow::field("wide_decimal", wide_decimal_type), + arrow::field("wide_uint64", arrow::uint64()), + }); + ASSERT_TRUE( + write_lance_record_batch( + dataset_uri, arrow::RecordBatch::Make(schema, 1, + {row_ids, half_floats, scaled_decimals, + wide_decimals, uint64_values})) + .ok()); + + const auto doris_wide_value = static_cast(wide_value); + const VExprContextSPtrs runtime_filters { + create_typed_runtime_in_conjunct("half_float", 1.5F, + std::make_shared(), 47), + create_typed_runtime_in_conjunct( + "scaled_decimal", Decimal32(12345), std::make_shared(9, 2), + 48), + create_typed_runtime_in_conjunct( + "wide_decimal", Decimal128V3(doris_wide_value), + std::make_shared(38, 0), 49), + create_typed_runtime_in_conjunct("wide_uint64", doris_wide_value, + std::make_shared(), 50), + }; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_unsupported_numeric_runtime_filter_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE( + init_reader(&reader, columns, &state, &profile, &scan_params, runtime_filters).ok()); + ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok()); + + Block block; + add_output_columns(&block, columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(1U, block.rows()); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderFilterTest, LeavesResidualPredicatesToScanner) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_scanner_residual_fixture"); + TFileScanRangeParams scan_params; + auto residual = VExprContext::create_shared(std::make_shared()); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params, {residual}).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + ASSERT_FALSE(reader.TEST_conjuncts_empty()); + + Block block; + add_output_columns(&block, columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_GT(block.rows(), 0); + EXPECT_TRUE(reader.close().ok()); +} + +void expect_filtered_row_ids(const char* profile_name, const std::string& substrait_filter_base64, + const std::vector& expected_row_ids) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile(profile_name); + TFileScanRangeParams scan_params; + std::string substrait_filter; + ASSERT_TRUE(base64_decode(substrait_filter_base64, &substrait_filter)); + TLanceScanParams lance_scan_params; + lance_scan_params.__set_lance_substrait_filter(std::move(substrait_filter)); + scan_params.__set_lance_scan_params(std::move(lance_scan_params)); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + ASSERT_TRUE(reader.TEST_conjuncts_empty()); + + Block block; + add_output_columns(&block, columns); + std::vector actual_row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& values = assert_cast(*block.get_by_position(0).column); + actual_row_ids.insert(actual_row_ids.end(), values.get_data().begin(), + values.get_data().end()); + } + auto sorted_expected_row_ids = expected_row_ids; + std::ranges::sort(actual_row_ids); + std::ranges::sort(sorted_expected_row_ids); + EXPECT_EQ(sorted_expected_row_ids, actual_row_ids); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderFilterTest, PushesDecimalFilterFromFeSubstrait) { + // Generated by the FE LancePredicateConverter for decimal_9_2 >= 0.00 + // against this fixture's complete Arrow schema. + const std::string substrait_filter_base64 = + "Ch4IARIaL2Z1bmN0aW9uc19jb21wYXJpc29uLnlhbWwSERoPCAEaC2d0ZTphbnlfYW55GkQKNRozGgQKAhACIg" + "waChIICgQS" + "AggJIgAiHRobChnCARYKEAAAAAAAAAAAAAAAAAAAAAAQCRgCGgtmaWx0ZXJfbWFzayKfBgoGcm93X2lkCgpib2" + "9sX3ZhbHVl" + "Cg10aW55aW50X3ZhbHVlCg5zbWFsbGludF92YWx1ZQoJaW50X3ZhbHVlCgxiaWdpbnRfdmFsdWUKC2Zsb2F0X3" + "ZhbHVlCgxk" + "b3VibGVfdmFsdWUKC2RlY2ltYWxfMV8wCgtkZWNpbWFsXzlfMgoMZGVjaW1hbF8xMF8wCgxkZWNpbWFsXzE4Xz" + "QKDGRlY2lt" + "YWxfMTlfMAoNZGVjaW1hbF8zOF8xMAokX191bmxpa2VseV9uYW1lX3BsYWNlaG9sZGVyX2RvcmlzXzE0CiRfX3" + "VubGlrZWx5" + "X25hbWVfcGxhY2Vob2xkZXJfZG9yaXNfMTUKCnRleHRfdmFsdWUKJF9fdW5saWtlbHlfbmFtZV9wbGFjZWhvbG" + "Rlcl9kb3Jp" + "c18xNwoKZGF0ZV92YWx1ZQokX191bmxpa2VseV9uYW1lX3BsYWNlaG9sZGVyX2RvcmlzXzE5CiRfX3VubGlrZW" + "x5X25hbWVf" + "cGxhY2Vob2xkZXJfZG9yaXNfMjAKJF9fdW5saWtlbHlfbmFtZV9wbGFjZWhvbGRlcl9kb3Jpc18yMQokX191bm" + "xpa2VseV9u" + "YW1lX3BsYWNlaG9sZGVyX2RvcmlzXzIyCiRfX3VubGlrZWx5X25hbWVfcGxhY2Vob2xkZXJfZG9yaXNfMjMKJF" + "9fdW5saWtl" + "bHlfbmFtZV9wbGFjZWhvbGRlcl9kb3Jpc18yNAoFbGFiZWwKJF9fdW5saWtlbHlfbmFtZV9wbGFjZWhvbGRlcl" + "9kb3Jpc18y" + "NhLHAQoEOgIQAgoECgIQAQoEEgIQAQoEGgIQAQoEKgIQAQoEOgIQAQoEUgIQAQoEWgIQAQoHwgEEEAEgAQoJwg" + "EGCAIQCSAB" + "CgfCAQQQCiABCgnCAQYIBBASIAEKB8IBBBATIAEKCcIBBggKECYgAQoF8gECGAEKBfIBAhgBCgRiAhABCgXyAQ" + "IYAQoFggEC" + "EAEKBfIBAhgBCgXyAQIYAQoF8gECGAEKBfIBAhgBCgXyAQIYAQoF8gECGAEKBGICEAEKBfIBAhgBGAI6DxBGKg" + "tkb3Jpcy1s" + "YW5jZQ=="; + expect_filtered_row_ids("lance_decimal_filter_fixture", substrait_filter_base64, {2, 4}); +} + +TEST(LanceTableReaderFilterTest, PushesDateFilterFromFeSubstrait) { + // Generated by the FE LancePredicateConverter for + // date_value >= DATE '1970-01-01' against this fixture's complete Arrow schema. + const std::string substrait_filter_base64 = + "Ch4IARIaL2Z1bmN0aW9uc19jb21wYXJpc29uLnlhbWwSERoPCAEaC2d0ZTphbnlfYW55Gi4KHxodGgQKAhACIg" + "waChIICgQS" + "AggSIgAiBxoFCgOAAQAaC2ZpbHRlcl9tYXNrIp8GCgZyb3dfaWQKCmJvb2xfdmFsdWUKDXRpbnlpbnRfdmFsdW" + "UKDnNtYWxs" + "aW50X3ZhbHVlCglpbnRfdmFsdWUKDGJpZ2ludF92YWx1ZQoLZmxvYXRfdmFsdWUKDGRvdWJsZV92YWx1ZQoLZG" + "VjaW1hbF8x" + "XzAKC2RlY2ltYWxfOV8yCgxkZWNpbWFsXzEwXzAKDGRlY2ltYWxfMThfNAoMZGVjaW1hbF8xOV8wCg1kZWNpbW" + "FsXzM4XzEw" + "CiRfX3VubGlrZWx5X25hbWVfcGxhY2Vob2xkZXJfZG9yaXNfMTQKJF9fdW5saWtlbHlfbmFtZV9wbGFjZWhvbG" + "Rlcl9kb3Jp" + "c18xNQoKdGV4dF92YWx1ZQokX191bmxpa2VseV9uYW1lX3BsYWNlaG9sZGVyX2RvcmlzXzE3CgpkYXRlX3ZhbH" + "VlCiRfX3Vu" + "bGlrZWx5X25hbWVfcGxhY2Vob2xkZXJfZG9yaXNfMTkKJF9fdW5saWtlbHlfbmFtZV9wbGFjZWhvbGRlcl9kb3" + "Jpc18yMAok" + "X191bmxpa2VseV9uYW1lX3BsYWNlaG9sZGVyX2RvcmlzXzIxCiRfX3VubGlrZWx5X25hbWVfcGxhY2Vob2xkZX" + "JfZG9yaXNf" + "MjIKJF9fdW5saWtlbHlfbmFtZV9wbGFjZWhvbGRlcl9kb3Jpc18yMwokX191bmxpa2VseV9uYW1lX3BsYWNlaG" + "9sZGVyX2Rv" + "cmlzXzI0CgVsYWJlbAokX191bmxpa2VseV9uYW1lX3BsYWNlaG9sZGVyX2RvcmlzXzI2EscBCgQ6AhACCgQKAh" + "ABCgQSAhAB" + "CgQaAhABCgQqAhABCgQ6AhABCgRSAhABCgRaAhABCgfCAQQQASABCgnCAQYIAhAJIAEKB8IBBBAKIAEKCcIBBg" + "gEEBIgAQoH" + "wgEEEBMgAQoJwgEGCAoQJiABCgXyAQIYAQoF8gECGAEKBGICEAEKBfIBAhgBCgWCAQIQAQoF8gECGAEKBfIBAh" + "gBCgXyAQIY" + "AQoF8gECGAEKBfIBAhgBCgXyAQIYAQoEYgIQAQoF8gECGAEYAjoPEEYqC2RvcmlzLWxhbmNl"; + expect_filtered_row_ids("lance_date_filter_fixture", substrait_filter_base64, {2, 3}); +} + +DataTypePtr nullable_type(PrimitiveType type, int precision = 0, int scale = 0) { + return DataTypeFactory::instance().create_data_type(type, true, precision, scale); +} + +// Creates an Arrow Duration array with one value and one null. +std::shared_ptr make_duration_array(arrow::TimeUnit::type unit, int64_t value) { + arrow::DurationBuilder builder(arrow::duration(unit), arrow::default_memory_pool()); + EXPECT_TRUE(builder.Append(value).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +// Creates a Lance BFloat16 array with one vector and one null. +std::shared_ptr make_bfloat16_vector_array() { + auto values = std::make_shared(arrow::fixed_size_binary(2)); + arrow::FixedSizeListBuilder builder(arrow::default_memory_pool(), values, 2); + EXPECT_TRUE(builder.Append().ok()); + const std::array one {0x80, 0x3F}; + const std::array two {0x00, 0x40}; + EXPECT_TRUE(values->Append(one.data()).ok()); + EXPECT_TRUE(values->Append(two.data()).ok()); + EXPECT_TRUE(builder.AppendNull().ok()); + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +// Creates a scalar Lance BFloat16 array from raw 16-bit values. +std::shared_ptr make_bfloat16_array( + const std::vector>& values) { + arrow::FixedSizeBinaryBuilder builder(arrow::fixed_size_binary(2)); + for (const auto& value : values) { + if (!value.has_value()) { + EXPECT_TRUE(builder.AppendNull().ok()); + continue; + } + std::array bytes {}; + LittleEndian::Store16(bytes.data(), *value); + EXPECT_TRUE(builder.Append(bytes.data()).ok()); + } + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +// Creates an Arrow Decimal256 array; std::nullopt appends NULL. Each value is written with exactly +// `scale` fraction digits, so Decimal256::FromString() returns the unscaled integer that the +// array stores: "-0.01" becomes -1 in a decimal256(18, 2) array. +std::shared_ptr make_decimal256_array( + int32_t precision, int32_t scale, const std::vector>& values) { + arrow::Decimal256Builder builder(arrow::decimal256(precision, scale)); + for (const auto& value : values) { + if (!value.has_value()) { + EXPECT_TRUE(builder.AppendNull().ok()); + continue; + } + const auto decimal = arrow::Decimal256::FromString(*value); + EXPECT_TRUE(decimal.ok()) << *value << ": " << decimal.status().ToString(); + EXPECT_TRUE(decimal.ok() && builder.Append(*decimal).ok()) << *value; + } + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +std::pair array_range(const ColumnArray& array, size_t row) { + const auto& offsets = array.get_offsets(); + return {row == 0 ? 0 : static_cast(offsets[row - 1]), + static_cast(offsets[row])}; +} + +std::pair map_range(const ColumnMap& map, size_t row) { + const auto& offsets = map.get_offsets(); + return {row == 0 ? 0 : static_cast(offsets[row - 1]), + static_cast(offsets[row])}; +} + +} // namespace + +TEST(LanceTableReaderSchemaTest, FetchesSchemaWithoutFragmentIdsOrScanInitialization) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + + // A local TVF schema request opens the latest dataset snapshot but returns only its schema. + auto range = make_latest_lance_range(dataset_uri); + + TFileScanRangeParams scan_params; + std::vector column_names; + std::vector column_types; + LanceTableReader reader; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &column_names, &column_types).ok()); + ASSERT_EQ(column_names.size(), column_types.size()); + + const auto row_id = std::find(column_names.begin(), column_names.end(), "row_id"); + ASSERT_NE(column_names.end(), row_id); + const auto row_id_idx = static_cast(std::distance(column_names.begin(), row_id)); + ASSERT_NE(nullptr, column_types[row_id_idx]); + EXPECT_EQ(TYPE_BIGINT, column_types[row_id_idx]->get_primitive_type()); + + const auto embedding = std::find(column_names.begin(), column_names.end(), "embedding"); + ASSERT_NE(column_names.end(), embedding); + const auto embedding_idx = static_cast(std::distance(column_names.begin(), embedding)); + ASSERT_NE(nullptr, column_types[embedding_idx]); + EXPECT_EQ(TYPE_ARRAY, column_types[embedding_idx]->get_primitive_type()); + + const auto binary_value = std::find(column_names.begin(), column_names.end(), "binary_value"); + ASSERT_NE(column_names.end(), binary_value); + const auto binary_idx = static_cast(std::distance(column_names.begin(), binary_value)); + ASSERT_NE(nullptr, column_types[binary_idx]); + const auto binary_type = remove_nullable(column_types[binary_idx]); + ASSERT_EQ(TYPE_VARBINARY, binary_type->get_primitive_type()); + EXPECT_EQ(std::numeric_limits::max(), + assert_cast(*binary_type).len()); +} + +// Verifies the additional mappings and preserves unknown extensions as unsupported. +TEST(LanceTableReaderSchemaTest, MapsAdditionalTypesAndPreservesUnknownExtensions) { + const auto unknown_extension_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"doris.test.extension"}); + const auto json_extension_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"arrow.json"}); + const auto bfloat16_extension_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto bfloat16_item = arrow::field("item", arrow::fixed_size_binary(2)) + ->WithMetadata(bfloat16_extension_metadata); + const auto arrow_schema = arrow::schema({ + arrow::field("row_id", arrow::int64()), + arrow::field("null_value", arrow::null()), + arrow::field("duration", arrow::duration(arrow::TimeUnit::MILLI)), + arrow::field("json", arrow::utf8())->WithMetadata(json_extension_metadata), + arrow::field("bfloat16_vector", arrow::fixed_size_list(bfloat16_item, 4)), + arrow::field("dictionary", arrow::dictionary(arrow::int16(), arrow::utf8())), + arrow::field("unknown_extension", arrow::utf8()) + ->WithMetadata(unknown_extension_metadata), + arrow::field("name", arrow::utf8()), + }); + + std::vector column_names; + std::vector column_types; + ASSERT_TRUE(convert_arrow_schema_to_doris(arrow_schema, &column_names, &column_types).ok()); + + EXPECT_EQ((std::vector {"row_id", "null_value", "duration", "json", + "bfloat16_vector", "dictionary", "unknown_extension", + "name"}), + column_names); + ASSERT_EQ(column_names.size(), column_types.size()); + for (const auto& column_type : column_types) { + ASSERT_NE(nullptr, column_type); + } + EXPECT_EQ(TYPE_BIGINT, column_types[0]->get_primitive_type()); + EXPECT_TRUE(column_types[1]->is_null_literal()); + EXPECT_EQ(TYPE_BIGINT, column_types[2]->get_primitive_type()); + EXPECT_EQ(TYPE_JSONB, column_types[3]->get_primitive_type()); + ASSERT_EQ(TYPE_ARRAY, column_types[4]->get_primitive_type()); + const auto& bfloat16_array = + assert_cast(*remove_nullable(column_types[4])); + EXPECT_EQ(TYPE_FLOAT, bfloat16_array.get_nested_type()->get_primitive_type()); + EXPECT_EQ(INVALID_TYPE, column_types[5]->get_primitive_type()); + EXPECT_EQ(INVALID_TYPE, column_types[6]->get_primitive_type()); + EXPECT_EQ(TYPE_STRING, column_types[7]->get_primitive_type()); +} + +// Verifies malformed storage for known extensions remains unsupported. +TEST(LanceTableReaderSchemaTest, RejectsMalformedKnownExtensionStorage) { + const auto json_extension_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"arrow.json"}); + const auto bfloat16_extension_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto arrow_schema = arrow::schema({ + arrow::field("json", arrow::binary())->WithMetadata(json_extension_metadata), + arrow::field("bfloat16", arrow::fixed_size_binary(4)) + ->WithMetadata(bfloat16_extension_metadata), + }); + + std::vector column_names; + std::vector column_types; + ASSERT_TRUE(convert_arrow_schema_to_doris(arrow_schema, &column_names, &column_types).ok()); + ASSERT_EQ(2, column_types.size()); + for (const auto& column_type : column_types) { + ASSERT_NE(nullptr, column_type); + EXPECT_EQ(INVALID_TYPE, column_type->get_primitive_type()); + } +} + +// Verifies values, nullability, and precision when reading the additional types. +TEST(LanceTableReaderTypeTest, ReadsAdditionalArrowAndLanceTypes) { + const auto json_extension_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"arrow.json"}); + const auto bfloat16_extension_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto bfloat16_item = arrow::field("item", arrow::fixed_size_binary(2)) + ->WithMetadata(bfloat16_extension_metadata); + const auto schema = arrow::schema({ + arrow::field("null_value", arrow::null()), + arrow::field("duration_s", arrow::duration(arrow::TimeUnit::SECOND)), + arrow::field("duration_ms", arrow::duration(arrow::TimeUnit::MILLI)), + arrow::field("duration_us", arrow::duration(arrow::TimeUnit::MICRO)), + arrow::field("duration_ns", arrow::duration(arrow::TimeUnit::NANO)), + arrow::field("json_value", arrow::utf8())->WithMetadata(json_extension_metadata), + arrow::field("bfloat16_vector", arrow::fixed_size_list(bfloat16_item, 2)), + }); + + arrow::StringBuilder json_builder; + ASSERT_TRUE(json_builder.Append(R"({"engine":"doris"})").ok()); + ASSERT_TRUE(json_builder.AppendNull().ok()); + std::shared_ptr json_array; + ASSERT_TRUE(json_builder.Finish(&json_array).ok()); + + const auto record_batch = + arrow::RecordBatch::Make(schema, 2, + { + std::make_shared(2), + make_duration_array(arrow::TimeUnit::SECOND, 1), + make_duration_array(arrow::TimeUnit::MILLI, 1000), + make_duration_array(arrow::TimeUnit::MICRO, 1000000), + make_duration_array(arrow::TimeUnit::NANO, 1000000000), + json_array, + make_bfloat16_vector_array(), + }); + + const auto bfloat16_array_type = + make_nullable(std::make_shared(nullable_type(TYPE_FLOAT))); + const Columns columns { + projected_column("null_value", nullable_type(TYPE_NULL)), + projected_column("duration_s", TYPE_BIGINT, true), + projected_column("duration_ms", TYPE_BIGINT, true), + projected_column("duration_us", TYPE_BIGINT, true), + projected_column("duration_ns", TYPE_BIGINT, true), + projected_column("json_value", TYPE_JSONB, true), + projected_column("bfloat16_vector", bfloat16_array_type), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_additional_types"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + ASSERT_TRUE(convert_record_batch_for_test(&state, columns, record_batch, &block, &rows).ok()); + ASSERT_EQ(2, rows); + + const auto& null_values = assert_cast(*block.get_by_position(0).column); + EXPECT_EQ((ColumnUInt8::Container {1, 1}), null_values.get_null_map_data()); + + const std::array expected_durations {1, 1000, 1000000, 1000000000}; + for (size_t column_idx = 0; column_idx < expected_durations.size(); ++column_idx) { + const auto& duration = + assert_cast(*block.get_by_position(column_idx + 1).column); + const auto& values = assert_cast(duration.get_nested_column()); + EXPECT_EQ(0, duration.get_null_map_data()[0]); + EXPECT_EQ(1, duration.get_null_map_data()[1]); + EXPECT_EQ(expected_durations[column_idx], values.get_data()[0]); + } + + const auto& json_values = assert_cast(*block.get_by_position(5).column); + EXPECT_EQ(0, json_values.get_null_map_data()[0]); + EXPECT_EQ(1, json_values.get_null_map_data()[1]); + EXPECT_EQ(R"({"engine":"doris"})", columns[5].type->to_string(json_values, 0)); + + const auto& vectors = assert_cast(*block.get_by_position(6).column); + EXPECT_EQ(0, vectors.get_null_map_data()[0]); + EXPECT_EQ(1, vectors.get_null_map_data()[1]); + const auto& vector_values = assert_cast(vectors.get_nested_column()); + EXPECT_EQ((ColumnArray::Offsets64 {2, 4}), vector_values.get_offsets()); + const auto& bfloat16_values = assert_cast(vector_values.get_data()); + const auto& floats = assert_cast(bfloat16_values.get_nested_column()); + EXPECT_FLOAT_EQ(1.0F, floats.get_data()[0]); + EXPECT_FLOAT_EQ(2.0F, floats.get_data()[1]); + EXPECT_TRUE(reader.close().ok()); +} + +// Verifies sliced scalar arrays are compacted before Doris reads rows from offset zero. +TEST(LanceTableReaderTypeTest, ReadsSlicedDurationAndJsonValues) { + const auto json_extension_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"arrow.json"}); + const auto duration_type = arrow::duration(arrow::TimeUnit::MILLI); + const auto schema = arrow::schema({ + arrow::field("duration", duration_type), + arrow::field("json_value", arrow::utf8())->WithMetadata(json_extension_metadata), + }); + + arrow::DurationBuilder duration_builder(duration_type, arrow::default_memory_pool()); + ASSERT_TRUE(duration_builder.Append(111).ok()); + ASSERT_TRUE(duration_builder.Append(222).ok()); + ASSERT_TRUE(duration_builder.AppendNull().ok()); + ASSERT_TRUE(duration_builder.Append(444).ok()); + std::shared_ptr durations; + ASSERT_TRUE(duration_builder.Finish(&durations).ok()); + + arrow::StringBuilder json_builder; + ASSERT_TRUE(json_builder.Append(R"({"sentinel":true})").ok()); + ASSERT_TRUE(json_builder.Append(R"({"row":1})").ok()); + ASSERT_TRUE(json_builder.Append(R"({"row":2})").ok()); + ASSERT_TRUE(json_builder.Append(R"({"tail":true})").ok()); + std::shared_ptr json_values; + ASSERT_TRUE(json_builder.Finish(&json_values).ok()); + + const auto record_batch = + arrow::RecordBatch::Make(schema, 2, {durations->Slice(1, 2), json_values->Slice(1, 2)}); + const Columns columns { + projected_column("duration", TYPE_BIGINT, true), + projected_column("json_value", TYPE_JSONB, true), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_sliced_scalars"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + ASSERT_TRUE(convert_record_batch_for_test(&state, columns, record_batch, &block, &rows).ok()); + ASSERT_EQ(2, rows); + + const auto& duration = assert_cast(*block.get_by_position(0).column); + const auto& duration_values = assert_cast(duration.get_nested_column()); + EXPECT_EQ((ColumnUInt8::Container {0, 1}), duration.get_null_map_data()); + EXPECT_EQ(222, duration_values.get_data()[0]); + + const auto& json = assert_cast(*block.get_by_position(1).column); + EXPECT_EQ((ColumnUInt8::Container {0, 0}), json.get_null_map_data()); + EXPECT_EQ(R"({"row":1})", columns[1].type->to_string(json, 0)); + EXPECT_EQ(R"({"row":2})", columns[1].type->to_string(json, 1)); + EXPECT_TRUE(reader.close().ok()); +} + +// Verifies sliced nested registered JSON arrays are unwrapped before parent compaction. +TEST(LanceTableReaderTypeTest, ReadsRegisteredJsonNestedInSlicedList) { + const auto json_type = arrow::extension::json(); + const auto item_field = arrow::field("item", json_type); + const auto list_type = arrow::list(item_field); + + arrow::StringBuilder json_builder; + ASSERT_TRUE(json_builder.Append(R"({"sentinel":true})").ok()); + ASSERT_TRUE(json_builder.Append(R"({"row":1})").ok()); + ASSERT_TRUE(json_builder.Append(R"({"row":2})").ok()); + ASSERT_TRUE(json_builder.Append(R"({"tail":true})").ok()); + std::shared_ptr json_storage; + ASSERT_TRUE(json_builder.Finish(&json_storage).ok()); + const auto json_values = arrow::ExtensionType::WrapArray(json_type, json_storage); + + arrow::Int32Builder offsets_builder; + ASSERT_TRUE(offsets_builder.AppendValues({0, 1, 3, 4}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offsets_builder.Finish(&offsets).ok()); + auto list_result = arrow::ListArray::FromArrays(list_type, *offsets, *json_values); + ASSERT_TRUE(list_result.ok()) << list_result.status().ToString(); + const auto input = std::move(list_result).ValueUnsafe()->Slice(1, 1); + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test(arrow::field("values", list_type), + input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto normalized_list = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, normalized_list); + EXPECT_EQ(0, normalized_list->offset()); + ASSERT_EQ(2, normalized_list->values()->length()); + const auto normalized_json = + std::dynamic_pointer_cast(normalized_list->values()); + ASSERT_NE(nullptr, normalized_json); + EXPECT_EQ(R"({"row":1})", normalized_json->GetString(0)); + EXPECT_EQ(R"({"row":2})", normalized_json->GetString(1)); + + const auto schema = arrow::schema({arrow::field("values", list_type)}); + const auto record_batch = arrow::RecordBatch::Make(schema, 1, {input}); + const auto doris_list_type = + make_nullable(std::make_shared(nullable_type(TYPE_JSONB))); + const Columns columns {projected_column("values", doris_list_type)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_sliced_nested_json"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + ASSERT_TRUE(convert_record_batch_for_test(&state, columns, record_batch, &block, &rows).ok()); + ASSERT_EQ(1, rows); + + const auto& nullable_list = + assert_cast(*block.get_by_position(0).column); + const auto& list = assert_cast(nullable_list.get_nested_column()); + EXPECT_EQ((ColumnArray::Offsets64 {2}), list.get_offsets()); + const auto& items = assert_cast(list.get_data()); + const auto json_item_type = nullable_type(TYPE_JSONB); + EXPECT_EQ(R"({"row":1})", json_item_type->to_string(items, 0)); + EXPECT_EQ(R"({"row":2})", json_item_type->to_string(items, 1)); + EXPECT_TRUE(reader.close().ok()); +} + +// Verifies BFloat16 special values retain their exact Float32 bit patterns. +TEST(LanceTableReaderTypeTest, ConvertsBFloat16SpecialValuesExactly) { + const auto metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto field = arrow::field("value", arrow::fixed_size_binary(2))->WithMetadata(metadata); + const std::vector> input_bits {0x0000, 0x8000, 0x3F80, 0xBF80, + 0x7F80, 0xFF80, 0x7FC1, std::nullopt}; + const std::array expected_bits {0x00000000, 0x80000000, 0x3F800000, 0xBF800000, + 0x7F800000, 0xFF800000, 0x7FC10000}; + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test(field, make_bfloat16_array(input_bits), + &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto floats = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, floats); + ASSERT_EQ(input_bits.size(), static_cast(floats->length())); + for (size_t index = 0; index < expected_bits.size(); ++index) { + ASSERT_FALSE(floats->IsNull(index)); + EXPECT_EQ(expected_bits[index], std::bit_cast(floats->Value(index))); + } + EXPECT_TRUE(floats->IsNull(expected_bits.size())); +} + +// Verifies normalization honors the supplied pool when widening and compacting arrays. +TEST(LanceTableReaderTypeTest, HonorsNormalizationMemoryPoolLimit) { + arrow::ProxyMemoryPool proxy_pool(arrow::default_memory_pool()); + arrow::CappedMemoryPool capped_pool(&proxy_pool, 0); + std::shared_ptr normalized; + + const auto bfloat16_metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto bfloat16_field = + arrow::field("value", arrow::fixed_size_binary(2))->WithMetadata(bfloat16_metadata); + const auto bfloat16_status = normalize_lance_arrow_array_for_test( + bfloat16_field, make_bfloat16_array({0x3F80}), &normalized, &capped_pool); + EXPECT_FALSE(bfloat16_status.ok()); + EXPECT_NE(std::string::npos, + bfloat16_status.to_string().find("reserve Lance BFloat16 output failed")); + + const auto duration_type = arrow::duration(arrow::TimeUnit::MILLI); + arrow::DurationBuilder duration_builder(duration_type, arrow::default_memory_pool()); + ASSERT_TRUE(duration_builder.AppendValues({100, 200}).ok()); + std::shared_ptr durations; + ASSERT_TRUE(duration_builder.Finish(&durations).ok()); + const auto duration_status = + normalize_lance_arrow_array_for_test(arrow::field("duration", duration_type), + durations->Slice(1, 1), &normalized, &capped_pool); + EXPECT_FALSE(duration_status.ok()); + EXPECT_NE(std::string::npos, + duration_status.to_string().find("reserve sliced Lance array builder failed")); +} + +// Verifies nested BFloat16 fields are converted without rebuilding unaffected sibling data. +TEST(LanceTableReaderTypeTest, ConvertsBFloat16NestedInStruct) { + const auto metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto value_field = + arrow::field("value", arrow::fixed_size_binary(2))->WithMetadata(metadata); + const auto id_field = arrow::field("id", arrow::int32()); + const auto bfloat16_values = make_bfloat16_array( + {std::optional {0x3F80}, std::optional {0xC020}}); + arrow::Int32Builder id_builder; + ASSERT_TRUE(id_builder.AppendValues({7, 8}).ok()); + std::shared_ptr ids; + ASSERT_TRUE(id_builder.Finish(&ids).ok()); + auto struct_result = arrow::StructArray::Make({bfloat16_values, ids}, {value_field, id_field}); + ASSERT_TRUE(struct_result.ok()) << struct_result.status().ToString(); + const auto input = std::move(struct_result).ValueUnsafe(); + const auto field = arrow::field("record", input->type()); + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test(field, input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(input.get(), normalized.get()); + const auto normalized_struct = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, normalized_struct); + const auto floats = std::dynamic_pointer_cast(normalized_struct->field(0)); + ASSERT_NE(nullptr, floats); + EXPECT_FLOAT_EQ(1.0F, floats->Value(0)); + EXPECT_FLOAT_EQ(-2.5F, floats->Value(1)); + EXPECT_EQ(ids->data().get(), normalized_struct->field(1)->data().get()); +} + +// Verifies sliced List normalization converts only values visible through parent offsets. +TEST(LanceTableReaderTypeTest, NormalizesVisibleBFloat16ValuesInSlicedList) { + const auto metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto item_field = + arrow::field("item", arrow::fixed_size_binary(2))->WithMetadata(metadata); + const auto values = make_bfloat16_array({0x3F80, 0x4000, 0x4040, 0x4080, 0x40A0, 0x40C0}); + arrow::Int32Builder offsets_builder; + ASSERT_TRUE(offsets_builder.AppendValues({0, 2, 5, 6}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offsets_builder.Finish(&offsets).ok()); + auto list_result = arrow::ListArray::FromArrays(arrow::list(item_field), *offsets, *values); + ASSERT_TRUE(list_result.ok()) << list_result.status().ToString(); + const auto input = std::move(list_result).ValueUnsafe()->Slice(1, 1); + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test( + arrow::field("values", arrow::list(item_field)), input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto list = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, list); + EXPECT_EQ(0, list->offset()); + ASSERT_EQ(1, list->length()); + EXPECT_EQ(0, list->value_offset(0)); + EXPECT_EQ(3, list->value_offset(1)); + const auto floats = std::dynamic_pointer_cast(list->values()); + ASSERT_NE(nullptr, floats); + ASSERT_EQ(3, floats->length()); + EXPECT_FLOAT_EQ(3.0F, floats->Value(0)); + EXPECT_FLOAT_EQ(4.0F, floats->Value(1)); + EXPECT_FLOAT_EQ(5.0F, floats->Value(2)); +} + +// Verifies sliced LargeList normalization rebases 64-bit offsets and trims child values. +TEST(LanceTableReaderTypeTest, NormalizesVisibleBFloat16ValuesInSlicedLargeList) { + const auto metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto item_field = + arrow::field("item", arrow::fixed_size_binary(2))->WithMetadata(metadata); + const auto values = make_bfloat16_array({0x3F80, 0x4000, 0x4040, 0x4080, 0x40A0, 0x40C0}); + arrow::Int64Builder offsets_builder; + ASSERT_TRUE(offsets_builder.AppendValues({0, 1, 3, 6}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offsets_builder.Finish(&offsets).ok()); + auto list_result = + arrow::LargeListArray::FromArrays(arrow::large_list(item_field), *offsets, *values); + ASSERT_TRUE(list_result.ok()) << list_result.status().ToString(); + const auto input = std::move(list_result).ValueUnsafe()->Slice(1, 1); + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test( + arrow::field("values", arrow::large_list(item_field)), input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto list = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, list); + EXPECT_EQ(0, list->offset()); + ASSERT_EQ(1, list->length()); + EXPECT_EQ(0, list->value_offset(0)); + EXPECT_EQ(2, list->value_offset(1)); + const auto floats = std::dynamic_pointer_cast(list->values()); + ASSERT_NE(nullptr, floats); + ASSERT_EQ(2, floats->length()); + EXPECT_FLOAT_EQ(2.0F, floats->Value(0)); + EXPECT_FLOAT_EQ(3.0F, floats->Value(1)); +} + +// Verifies sliced FixedSizeList normalization trims values using the fixed child width. +TEST(LanceTableReaderTypeTest, NormalizesVisibleBFloat16ValuesInSlicedFixedSizeList) { + const auto metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto item_field = + arrow::field("item", arrow::fixed_size_binary(2))->WithMetadata(metadata); + const auto values = + make_bfloat16_array({0x3F80, 0x4000, 0x4040, 0x4080, 0x40A0, 0x40C0, 0x40E0, 0x4100}); + auto list_result = arrow::FixedSizeListArray::FromArrays(values, 2); + ASSERT_TRUE(list_result.ok()) << list_result.status().ToString(); + const auto input = std::move(list_result).ValueUnsafe()->Slice(2, 1); + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test( + arrow::field("values", arrow::fixed_size_list(item_field, 2)), input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto list = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, list); + EXPECT_EQ(0, list->offset()); + ASSERT_EQ(1, list->length()); + const auto floats = std::dynamic_pointer_cast(list->values()); + ASSERT_NE(nullptr, floats); + ASSERT_EQ(2, floats->length()); + EXPECT_FLOAT_EQ(5.0F, floats->Value(0)); + EXPECT_FLOAT_EQ(6.0F, floats->Value(1)); +} + +// Verifies sliced Map normalization trims entries while preserving visible keys and values. +TEST(LanceTableReaderTypeTest, NormalizesVisibleBFloat16ValuesInSlicedMap) { + const auto metadata = + arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.bfloat16"}); + const auto item_field = + arrow::field("value", arrow::fixed_size_binary(2))->WithMetadata(metadata); + const auto values = make_bfloat16_array({0x3F80, 0x4000, 0x4040, 0x4080, 0x40A0, 0x40C0}); + arrow::Int32Builder offsets_builder; + ASSERT_TRUE(offsets_builder.AppendValues({0, 2, 5, 6}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offsets_builder.Finish(&offsets).ok()); + arrow::StringBuilder keys_builder; + ASSERT_TRUE(keys_builder.AppendValues({"k0", "k1", "k2", "k3", "k4", "k5"}).ok()); + std::shared_ptr keys; + ASSERT_TRUE(keys_builder.Finish(&keys).ok()); + const auto map_type = arrow::map(arrow::utf8(), item_field); + auto map_result = arrow::MapArray::FromArrays(map_type, offsets, keys, values); + ASSERT_TRUE(map_result.ok()) << map_result.status().ToString(); + const auto input = std::move(map_result).ValueUnsafe()->Slice(1, 1); + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test(arrow::field("values", map_type), + input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto map = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, map); + EXPECT_EQ(0, map->offset()); + ASSERT_EQ(1, map->length()); + EXPECT_EQ(0, map->value_offset(0)); + EXPECT_EQ(3, map->value_offset(1)); + const auto normalized_keys = std::dynamic_pointer_cast(map->keys()); + ASSERT_NE(nullptr, normalized_keys); + ASSERT_EQ(3, normalized_keys->length()); + EXPECT_EQ("k2", normalized_keys->GetString(0)); + EXPECT_EQ("k3", normalized_keys->GetString(1)); + EXPECT_EQ("k4", normalized_keys->GetString(2)); + const auto floats = std::dynamic_pointer_cast(map->items()); + ASSERT_NE(nullptr, floats); + ASSERT_EQ(3, floats->length()); + EXPECT_FLOAT_EQ(3.0F, floats->Value(0)); + EXPECT_FLOAT_EQ(4.0F, floats->Value(1)); + EXPECT_FLOAT_EQ(5.0F, floats->Value(2)); + + const auto schema = arrow::schema({arrow::field("values", map_type)}); + const auto record_batch = arrow::RecordBatch::Make(schema, 1, {input}); + const auto doris_map_type = make_nullable( + std::make_shared(nullable_type(TYPE_STRING), nullable_type(TYPE_FLOAT))); + const Columns columns {projected_column("values", doris_map_type)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_sliced_map"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + ASSERT_TRUE(convert_record_batch_for_test(&state, columns, record_batch, &block, &rows).ok()); + ASSERT_EQ(1, rows); + + const auto& nullable_map = assert_cast(*block.get_by_position(0).column); + const auto& materialized_map = assert_cast(nullable_map.get_nested_column()); + EXPECT_EQ((ColumnArray::Offsets64 {3}), materialized_map.get_offsets()); + const auto& materialized_keys = assert_cast(materialized_map.get_keys()); + const auto& key_values = + assert_cast(materialized_keys.get_nested_column()); + const auto& materialized_items = + assert_cast(materialized_map.get_values()); + const auto& item_values = + assert_cast(materialized_items.get_nested_column()); + ASSERT_EQ(3, key_values.size()); + ASSERT_EQ(3, item_values.size()); + EXPECT_EQ("k2", key_values.get_data_at(0).to_string()); + EXPECT_EQ("k3", key_values.get_data_at(1).to_string()); + EXPECT_EQ("k4", key_values.get_data_at(2).to_string()); + EXPECT_FLOAT_EQ(3.0F, item_values.get_data()[0]); + EXPECT_FLOAT_EQ(4.0F, item_values.get_data()[1]); + EXPECT_FLOAT_EQ(5.0F, item_values.get_data()[2]); + EXPECT_TRUE(reader.close().ok()); +} + +// Verifies the Lance JSON extension reads LargeBinary values and nulls through JSON SerDe. +TEST(LanceTableReaderTypeTest, ReadsLanceJsonLargeBinaryValues) { + const auto metadata = arrow::KeyValueMetadata::Make({"ARROW:extension:name"}, {"lance.json"}); + const auto schema = arrow::schema( + {arrow::field("json_value", arrow::large_binary())->WithMetadata(metadata)}); + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(R"({"format":"lance","value":42})").ok()); + ASSERT_TRUE(builder.AppendNull().ok()); + std::shared_ptr values; + ASSERT_TRUE(builder.Finish(&values).ok()); + const auto record_batch = arrow::RecordBatch::Make(schema, 2, {values}); + + const Columns columns {projected_column("json_value", TYPE_JSONB, true)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_json_large_binary"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + ASSERT_TRUE(convert_record_batch_for_test(&state, columns, record_batch, &block, &rows).ok()); + ASSERT_EQ(2, rows); + const auto& json_values = assert_cast(*block.get_by_position(0).column); + EXPECT_EQ((ColumnUInt8::Container {0, 1}), json_values.get_null_map_data()); + EXPECT_EQ(R"({"format":"lance","value":42})", columns[0].type->to_string(json_values, 0)); + EXPECT_TRUE(reader.close().ok()); +} + +// Verifies Duration values preserve signed 64-bit boundaries and nullability. +TEST(LanceTableReaderTypeTest, ReadsDurationBoundaryValues) { + const auto duration_type = arrow::duration(arrow::TimeUnit::NANO); + const auto schema = arrow::schema({arrow::field("duration", duration_type)}); + arrow::DurationBuilder builder(duration_type, arrow::default_memory_pool()); + const std::array expected {std::numeric_limits::min(), -1, 0, + std::numeric_limits::max()}; + ASSERT_TRUE(builder.AppendValues(expected.data(), expected.size()).ok()); + ASSERT_TRUE(builder.AppendNull().ok()); + std::shared_ptr values; + ASSERT_TRUE(builder.Finish(&values).ok()); + const auto record_batch = arrow::RecordBatch::Make(schema, 5, {values}); + + const Columns columns {projected_column("duration", TYPE_BIGINT, true)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_duration_boundaries"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + ASSERT_TRUE(convert_record_batch_for_test(&state, columns, record_batch, &block, &rows).ok()); + ASSERT_EQ(5, rows); + const auto& durations = assert_cast(*block.get_by_position(0).column); + const auto& duration_values = assert_cast(durations.get_nested_column()); + EXPECT_EQ((ColumnUInt8::Container {0, 0, 0, 0, 1}), durations.get_null_map_data()); + for (size_t index = 0; index < expected.size(); ++index) { + EXPECT_EQ(expected[index], duration_values.get_data()[index]); + } + EXPECT_TRUE(reader.close().ok()); +} + +// Verifies Date32 days after 9999-12-31 fail instead of reading as a non-null 0000-00-00, also +// when ARRAY, STRUCT, and MAP leaves reach the DATE SerDe through their element SerDes. +TEST(LanceTableReaderTypeTest, RejectsDate32OutsideDorisDateRange) { + // 1970-01-01 + 2932897 days is 10000-01-01. + arrow::Date32Builder date_builder; + ASSERT_TRUE(date_builder.Append(2932897).ok()); + std::shared_ptr dates; + ASSERT_TRUE(date_builder.Finish(&dates).ok()); + arrow::Int32Builder offsets_builder; + ASSERT_TRUE(offsets_builder.AppendValues({0, 1}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offsets_builder.Finish(&offsets).ok()); + arrow::StringBuilder keys_builder; + ASSERT_TRUE(keys_builder.Append("k").ok()); + std::shared_ptr keys; + ASSERT_TRUE(keys_builder.Finish(&keys).ok()); + + auto list_result = arrow::ListArray::FromArrays( + arrow::list(arrow::field("item", arrow::date32())), *offsets, *dates); + ASSERT_TRUE(list_result.ok()) << list_result.status().ToString(); + auto struct_result = arrow::StructArray::Make({dates}, {arrow::field("d", arrow::date32())}); + ASSERT_TRUE(struct_result.ok()) << struct_result.status().ToString(); + auto map_result = arrow::MapArray::FromArrays(arrow::map(arrow::utf8(), arrow::date32()), + offsets, keys, dates); + ASSERT_TRUE(map_result.ok()) << map_result.status().ToString(); + + const auto date_type = nullable_type(TYPE_DATEV2); + const std::vector, DataTypePtr>> cases { + {dates, date_type}, + {std::move(list_result).ValueUnsafe(), + make_nullable(std::make_shared(date_type))}, + {std::move(struct_result).ValueUnsafe(), + make_nullable(std::make_shared(DataTypes {date_type}, Strings {"d"}))}, + {std::move(map_result).ValueUnsafe(), + make_nullable(std::make_shared(nullable_type(TYPE_STRING), date_type))}, + }; + for (const auto& [array, doris_type] : cases) { + SCOPED_TRACE(array->type()->ToString()); + const auto schema = arrow::schema({arrow::field("value", array->type())}); + const auto record_batch = arrow::RecordBatch::Make(schema, 1, {array}); + const Columns columns {projected_column("value", doris_type)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + const auto status = + convert_record_batch_for_test(&state, columns, record_batch, &block, &rows); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, + status.to_string().find("Arrow Date32 value is outside the Doris DATE range")) + << status.to_string(); + } +} + +// Verifies the last Doris DATE reads exactly and the payload under a NULL slot is not decoded. +TEST(LanceTableReaderTypeTest, ReadsLastDorisDateAndIgnoresNullDate32Payload) { + arrow::Date32Builder builder; + ASSERT_TRUE(builder.AppendValues({2932896, 2932897}).ok()); + std::shared_ptr payload; + ASSERT_TRUE(builder.Finish(&payload).ok()); + // Validity bitmap 0b01: row 0 holds 9999-12-31, row 1 is NULL over 10000-01-01. + auto data = payload->data()->Copy(); + data->buffers[0] = arrow::Buffer::FromVector(std::vector {0x01}); + data->SetNullCount(1); + const auto schema = arrow::schema({arrow::field("value", arrow::date32())}); + const auto record_batch = arrow::RecordBatch::Make(schema, 2, {arrow::MakeArray(data)}); + + const auto date_type = nullable_type(TYPE_DATEV2); + const Columns columns {projected_column("value", date_type)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + const auto status = convert_record_batch_for_test(&state, columns, record_batch, &block, &rows); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(2, rows); + const auto& values = assert_cast(*block.get_by_position(0).column); + EXPECT_EQ((ColumnUInt8::Container {0, 1}), values.get_null_map_data()); + EXPECT_EQ("9999-12-31", date_type->to_string(values, 0)); +} + +// Verifies ordinary nested columns bypass Arrow reconstruction when no BFloat16 exists. +TEST(LanceTableReaderTypeTest, KeepsOrdinaryNestedArrayUnchangedDuringNormalization) { + const auto nested_type = arrow::struct_({ + arrow::field("items", + arrow::list(arrow::field( + "item", arrow::struct_({arrow::field("value", arrow::int64())})))), + arrow::field("metadata", arrow::struct_({arrow::field("name", arrow::utf8()), + arrow::field("enabled", arrow::boolean())})), + }); + const auto field = arrow::field("ordinary_nested", nested_type); + auto array_result = arrow::MakeArrayOfNull(nested_type, 3); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + const auto array = std::move(array_result).ValueUnsafe(); + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test(field, array, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(array.get(), normalized.get()); +} + +// Verifies a Decimal256 field whose precision fits Decimal128 is narrowed with unchanged values, +// because Doris types it DECIMAL32/64/128I and those SerDes read only Decimal128Array. +TEST(LanceTableReaderTypeTest, NarrowsDecimal256WithDecimal128PrecisionDuringNormalization) { + const auto input = make_decimal256_array( + 18, 2, {"9999999999999999.99", "-9999999999999999.99", std::nullopt}); + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test( + arrow::field("amount", arrow::decimal256(18, 2)), input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(normalized->type()->Equals(arrow::decimal128(18, 2))) + << normalized->type()->ToString(); + const auto decimals = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, decimals); + ASSERT_EQ(3, decimals->length()); + EXPECT_EQ("9999999999999999.99", decimals->FormatValue(0)); + EXPECT_EQ("-9999999999999999.99", decimals->FormatValue(1)); + EXPECT_TRUE(decimals->IsNull(2)); +} + +// Verifies a Decimal256 field above Decimal128 precision keeps its array for DECIMAL256. +TEST(LanceTableReaderTypeTest, KeepsWideDecimal256UnchangedDuringNormalization) { + const auto input = make_decimal256_array( + 39, 4, {"99999999999999999999999999999999999.9999", std::nullopt}); + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test( + arrow::field("wide_amount", arrow::decimal256(39, 4)), input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(input.get(), normalized.get()); +} + +// Verifies narrowed Decimal256 values reach DECIMAL32, DECIMAL64, and DECIMAL128I unchanged. +TEST(LanceTableReaderTypeTest, ConvertsDecimal256WithDecimal128PrecisionToDorisDecimals) { + struct DecimalCase { + const char* name; + PrimitiveType doris_type; + int precision; + int scale; + std::array values; + }; + const std::array cases {{ + {"decimal_9_2", TYPE_DECIMAL32, 9, 2, {"9999999.99", "-9999999.99"}}, + {"decimal_18_2", TYPE_DECIMAL64, 18, 2, {"9999999999999999.99", "-0.01"}}, + {"decimal_38_10", + TYPE_DECIMAL128I, + 38, + 10, + {"9999999999999999999999999999.9999999999", + "-9999999999999999999999999999.9999999999"}}, + }}; + arrow::FieldVector fields; + std::vector> arrays; + Columns columns; + for (const auto& decimal_case : cases) { + fields.emplace_back(arrow::field( + decimal_case.name, arrow::decimal256(decimal_case.precision, decimal_case.scale))); + arrays.emplace_back(make_decimal256_array( + decimal_case.precision, decimal_case.scale, + {decimal_case.values[0], decimal_case.values[1], std::nullopt})); + columns.emplace_back(projected_column( + decimal_case.name, nullable_type(decimal_case.doris_type, decimal_case.precision, + decimal_case.scale))); + } + const auto record_batch = arrow::RecordBatch::Make(arrow::schema(fields), 3, arrays); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + const auto status = convert_record_batch_for_test(&state, columns, record_batch, &block, &rows); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(3, rows); + for (size_t column = 0; column < cases.size(); ++column) { + SCOPED_TRACE(cases[column].name); + const auto& decimals = + assert_cast(*block.get_by_position(column).column); + EXPECT_EQ((ColumnUInt8::Container {0, 0, 1}), decimals.get_null_map_data()); + EXPECT_EQ(std::string(cases[column].values[0]), + columns[column].type->to_string(decimals, 0)); + EXPECT_EQ(std::string(cases[column].values[1]), + columns[column].type->to_string(decimals, 1)); + } +} + +// Verifies Decimal256 leaves are narrowed inside sliced List, Map, and Struct arrays. The parent +// type is rebuilt around the narrowed child before compaction keeps only the visible values. +TEST(LanceTableReaderTypeTest, NarrowsDecimal256LeavesInSlicedNestedArrays) { + const auto decimal_type = arrow::decimal256(18, 2); + const auto values = make_decimal256_array(18, 2, {"1.00", "2.50", "-3.75", "4.00"}); + arrow::Int32Builder offsets_builder; + ASSERT_TRUE(offsets_builder.AppendValues({0, 1, 3, 4}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offsets_builder.Finish(&offsets).ok()); + arrow::StringBuilder keys_builder; + ASSERT_TRUE(keys_builder.AppendValues({"k0", "k1", "k2", "k3"}).ok()); + std::shared_ptr keys; + ASSERT_TRUE(keys_builder.Finish(&keys).ok()); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({10, 11, 12, 13}).ok()); + std::shared_ptr ids; + ASSERT_TRUE(ids_builder.Finish(&ids).ok()); + + // Offsets {0, 1, 3, 4}: list and map row 1 cover the child values 2.50 and -3.75. Struct row 2 + // holds -3.75 and id 12. + const auto list_type = arrow::list(arrow::field("item", decimal_type)); + auto list_result = arrow::ListArray::FromArrays(list_type, *offsets, *values); + ASSERT_TRUE(list_result.ok()) << list_result.status().ToString(); + const auto map_type = arrow::map(arrow::utf8(), decimal_type); + auto map_result = arrow::MapArray::FromArrays(map_type, offsets, keys, values); + ASSERT_TRUE(map_result.ok()) << map_result.status().ToString(); + auto struct_result = arrow::StructArray::Make( + {values, ids}, + {arrow::field("amount", decimal_type), arrow::field("id", arrow::int32())}); + ASSERT_TRUE(struct_result.ok()) << struct_result.status().ToString(); + const auto list_input = std::move(list_result).ValueUnsafe()->Slice(1, 1); + const auto map_input = std::move(map_result).ValueUnsafe()->Slice(1, 1); + const auto struct_input = std::move(struct_result).ValueUnsafe()->Slice(2, 1); + + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test(arrow::field("values", list_type), + list_input, &normalized); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto normalized_list = std::dynamic_pointer_cast(normalized); + ASSERT_NE(nullptr, normalized_list); + EXPECT_EQ(0, normalized_list->offset()); + EXPECT_TRUE(normalized_list->value_type()->Equals(arrow::decimal128(18, 2))) + << normalized_list->value_type()->ToString(); + EXPECT_EQ(2, normalized_list->values()->length()); + + const auto schema = arrow::schema({ + arrow::field("list_value", list_type), + arrow::field("map_value", map_type), + arrow::field("struct_value", struct_input->type()), + }); + const auto record_batch = + arrow::RecordBatch::Make(schema, 1, {list_input, map_input, struct_input}); + const auto decimal_doris_type = nullable_type(TYPE_DECIMAL64, 18, 2); + const Columns columns { + projected_column("list_value", + make_nullable(std::make_shared(decimal_doris_type))), + projected_column("map_value", make_nullable(std::make_shared( + nullable_type(TYPE_STRING), decimal_doris_type))), + projected_column("struct_value", + make_nullable(std::make_shared( + DataTypes {decimal_doris_type, nullable_type(TYPE_INT)}, + Strings {"amount", "id"}))), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + Block block; + add_output_columns(&block, columns); + size_t rows = 0; + const auto convert_status = + convert_record_batch_for_test(&state, columns, record_batch, &block, &rows); + ASSERT_TRUE(convert_status.ok()) << convert_status.to_string(); + ASSERT_EQ(1, rows); + + const auto& list_column = assert_cast( + assert_cast(*block.get_by_position(0).column) + .get_nested_column()); + EXPECT_EQ((ColumnArray::Offsets64 {2}), list_column.get_offsets()); + EXPECT_EQ("2.50", decimal_doris_type->to_string(list_column.get_data(), 0)); + EXPECT_EQ("-3.75", decimal_doris_type->to_string(list_column.get_data(), 1)); + + const auto& map_column = assert_cast( + assert_cast(*block.get_by_position(1).column) + .get_nested_column()); + EXPECT_EQ((ColumnArray::Offsets64 {2}), map_column.get_offsets()); + EXPECT_EQ("k1", nullable_type(TYPE_STRING)->to_string(map_column.get_keys(), 0)); + EXPECT_EQ("k2", nullable_type(TYPE_STRING)->to_string(map_column.get_keys(), 1)); + EXPECT_EQ("2.50", decimal_doris_type->to_string(map_column.get_values(), 0)); + EXPECT_EQ("-3.75", decimal_doris_type->to_string(map_column.get_values(), 1)); + + const auto& struct_column = assert_cast( + assert_cast(*block.get_by_position(2).column) + .get_nested_column()); + EXPECT_EQ("-3.75", decimal_doris_type->to_string(struct_column.get_column(0), 0)); + EXPECT_EQ("12", nullable_type(TYPE_INT)->to_string(struct_column.get_column(1), 0)); +} + +// Verifies a value outside the declared precision fails with the field name instead of being +// truncated: unscaled 10^18 does not fit decimal(18, 2), nor 10^38 decimal(38, 0). +TEST(LanceTableReaderTypeTest, RejectsDecimal256ValuesOutsideDeclaredPrecision) { + const std::array, 2> cases {{ + {"amount", 18, 2, "10000000000000000.00"}, + {"wide_amount", 38, 0, "100000000000000000000000000000000000000"}, + }}; + for (const auto& [name, precision, scale, value] : cases) { + SCOPED_TRACE(name); + std::shared_ptr normalized; + const auto status = normalize_lance_arrow_array_for_test( + arrow::field(name, arrow::decimal256(precision, scale)), + make_decimal256_array(precision, scale, {value}), &normalized); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find(std::string("field '") + name + "'")) + << status.to_string(); + EXPECT_NE(std::string::npos, status.to_string().find("does not fit in precision")) + << status.to_string(); + } +} + +// Verifies Lance keeps the 256-bit carrier of decimal256(18, 2) on disk, and the reader returns +// it as the DECIMAL64 column that the precision selects. +TEST(LanceTableReaderTypeTest, ReadsDecimal256WithDecimal128PrecisionFromLanceDataset) { + const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + const auto dataset_uri = + std::filesystem::temp_directory_path() / + ("doris_lance_narrow_decimal256_" + std::to_string(unique_suffix) + ".lance"); + Defer cleanup {[&] { + std::error_code error; + std::filesystem::remove_all(dataset_uri, error); + }}; + + arrow::Int64Builder row_id_builder; + ASSERT_TRUE(row_id_builder.AppendValues({1, 2, 3}).ok()); + std::shared_ptr row_ids; + ASSERT_TRUE(row_id_builder.Finish(&row_ids).ok()); + const auto amounts = + make_decimal256_array(18, 2, {"9999999999999999.99", "-0.01", std::nullopt}); + const auto schema = arrow::schema({arrow::field("row_id", arrow::int64()), + arrow::field("amount", arrow::decimal256(18, 2))}); + ASSERT_TRUE(write_lance_record_batch(dataset_uri, + arrow::RecordBatch::Make(schema, 3, {row_ids, amounts})) + .ok()); + + std::unique_ptr dataset( + lance_dataset_open(dataset_uri.c_str(), nullptr, 0), lance_dataset_close); + ASSERT_NE(nullptr, dataset); + std::shared_ptr stored_schema; + ASSERT_TRUE(import_lance_dataset_schema(dataset.get(), &stored_schema).ok()); + const auto stored_amount = stored_schema->GetFieldByName("amount"); + ASSERT_NE(nullptr, stored_amount); + EXPECT_EQ(arrow::Type::DECIMAL256, stored_amount->type()->id()); + + TFileScanRangeParams scan_params; + std::vector column_names; + std::vector column_types; + LanceTableReader schema_reader; + ASSERT_TRUE(schema_reader + .fetch_schema(make_latest_lance_range(dataset_uri), scan_params, + &column_names, &column_types) + .ok()); + ASSERT_EQ((std::vector {"row_id", "amount"}), column_names); + ASSERT_EQ(2, column_types.size()); + EXPECT_EQ(TYPE_DECIMAL64, remove_nullable(column_types[1])->get_primitive_type()); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("amount", column_types[1]), + }; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_narrow_decimal256"); + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok()); + + Block block; + add_output_columns(&block, columns); + const std::map> expected { + {1, "9999999999999999.99"}, {2, "-0.01"}, {3, std::nullopt}}; + std::set seen_row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& block_row_ids = + assert_cast(*block.get_by_position(0).column); + const auto& block_amounts = + assert_cast(*block.get_by_position(1).column); + for (size_t row = 0; row < block.rows(); ++row) { + const auto row_id = block_row_ids.get_data()[row]; + ASSERT_TRUE(expected.contains(row_id)) << row_id; + EXPECT_TRUE(seen_row_ids.emplace(row_id).second) << row_id; + const auto& expected_amount = expected.at(row_id); + EXPECT_EQ(!expected_amount.has_value(), block_amounts.is_null_at(row)) << row_id; + if (expected_amount.has_value()) { + EXPECT_EQ(*expected_amount, columns[1].type->to_string(block_amounts, row)); + } + } + } + EXPECT_EQ(3U, seen_row_ids.size()); + EXPECT_TRUE(reader.close().ok()); +} + +// Verifies the additional types through the full scan path. +TEST(LanceTableReaderTypeTest, ReadsAdditionalTypesFromCompatibilityFixture) { + const std::filesystem::path dataset_uri = + "./docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/" + "all_types.lance"; + const auto bfloat16_array_type = + make_nullable(std::make_shared(nullable_type(TYPE_FLOAT))); + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("null_col", nullable_type(TYPE_NULL)), + projected_column("duration_s_col", TYPE_BIGINT, true), + projected_column("duration_ms_col", TYPE_BIGINT, true), + projected_column("duration_us_col", TYPE_BIGINT, true), + projected_column("duration_ns_col", TYPE_BIGINT, true), + projected_column("json_col", TYPE_JSONB, true), + projected_column("bfloat16_vector_col", bfloat16_array_type), + }; + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_additional_types_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok()); + + Block block; + add_output_columns(&block, columns); + bool found = false; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < block.rows(); ++row) { + const auto& null_values = + assert_cast(*block.get_by_position(1).column); + EXPECT_EQ(1, null_values.get_null_map_data()[row]); + if (row_ids.get_data()[row] != 1) { + continue; + } + found = true; + const std::array expected_durations {1, 1000, 1000000, 1000000000}; + for (size_t duration_idx = 0; duration_idx < expected_durations.size(); + ++duration_idx) { + const auto& duration = assert_cast( + *block.get_by_position(duration_idx + 2).column); + const auto& values = assert_cast(duration.get_nested_column()); + EXPECT_EQ(0, duration.get_null_map_data()[row]); + EXPECT_EQ(expected_durations[duration_idx], values.get_data()[row]); + } + + const auto& json_values = + assert_cast(*block.get_by_position(6).column); + EXPECT_EQ(R"({"engine":"doris","format":"lance"})", + columns[6].type->to_string(json_values, row)); + + const auto& vectors = + assert_cast(*block.get_by_position(7).column); + const auto& vector_values = + assert_cast(vectors.get_nested_column()); + const auto [begin, end] = array_range(vector_values, row); + ASSERT_EQ(4, end - begin); + const auto& nullable_values = + assert_cast(vector_values.get_data()); + const auto& floats = + assert_cast(nullable_values.get_nested_column()); + for (size_t index = 0; index < 4; ++index) { + EXPECT_FLOAT_EQ(static_cast(index + 1), floats.get_data()[begin + index]); + } + } + } + EXPECT_TRUE(found); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderSchemaTest, FetchesNegativeScaleDecimalAsUnsupported) { + const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + const auto dataset_uri = + std::filesystem::temp_directory_path() / + ("doris_lance_negative_decimal_scale_" + std::to_string(unique_suffix) + ".lance"); + Defer cleanup {[&] { + std::error_code error; + std::filesystem::remove_all(dataset_uri, error); + }}; + + const auto decimal_type = arrow::decimal128(10, -2); + const auto arrow_schema = arrow::schema({arrow::field("negative_scale_decimal", decimal_type)}); + auto array = arrow::MakeArrayOfNull(decimal_type, 1); + ASSERT_TRUE(array.ok()) << array.status().ToString(); + const auto batch = arrow::RecordBatch::Make(arrow_schema, 1, {std::move(array).ValueUnsafe()}); + auto batch_reader = arrow::RecordBatchReader::Make({batch}, arrow_schema); + ASSERT_TRUE(batch_reader.ok()) << batch_reader.status().ToString(); + + ArrowArrayStream stream {}; + const auto export_status = + arrow::ExportRecordBatchReader(std::move(batch_reader).ValueUnsafe(), &stream); + ASSERT_TRUE(export_status.ok()) << export_status.ToString(); + static_cast( + ::lance::Dataset::write(dataset_uri.string(), &stream, ::lance::WriteMode::Create)); + + TFileScanRangeParams scan_params; + std::vector column_names; + std::vector column_types; + LanceTableReader reader; + const auto status = reader.fetch_schema(make_latest_lance_range(dataset_uri), scan_params, + &column_names, &column_types); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ((std::vector {"negative_scale_decimal"}), column_names); + ASSERT_EQ(1, column_types.size()); + ASSERT_NE(nullptr, column_types[0]); + EXPECT_EQ(INVALID_TYPE, column_types[0]->get_primitive_type()); +} + +TEST(LanceTableReaderScanTest, ReadsLatestSnapshotWithoutFragmentIds) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryOptions query_options; + query_options.__set_batch_size(2); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_latest_all_fragments_fixture"); + TFileScanRangeParams scan_params; + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok()); + + Block block; + add_output_columns(&block, columns); + std::array seen_rows {}; + size_t total_rows = 0; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < block.rows(); ++row) { + const auto row_id = row_ids.get_data()[row]; + ASSERT_GE(row_id, 1); + ASSERT_LE(row_id, 4); + EXPECT_FALSE(seen_rows[row_id]); + seen_rows[row_id] = true; + } + total_rows += block.rows(); + } + EXPECT_EQ(4U, total_rows); + EXPECT_TRUE(seen_rows[1]); + EXPECT_TRUE(seen_rows[2]); + EXPECT_TRUE(seen_rows[3]); + EXPECT_TRUE(seen_rows[4]); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderScanTest, RejectsStorageOptionWithEmbeddedNul) { + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + RuntimeProfile profile("lance_storage_option_embedded_nul"); + + // lance-c reads these as C strings, so a NUL truncates the option here while the FE goes on + // using the whole thing, leaving the two halves opening the dataset with different + // configuration. Dropping it instead of failing would only move that divergence. + TFileScanRangeParams scan_params; + TLanceScanParams lance_scan_params; + lance_scan_params.__set_lance_storage_options( + {{std::string("aws_region\0ignored", 18), "us-east-1"}}); + scan_params.__set_lance_scan_params(std::move(lance_scan_params)); + + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + + const auto status = prepare_range(&reader, make_latest_lance_range(dataset_uri)); + + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("contains a NUL"), std::string::npos); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderTypeTest, ReadsNumericTypesFromAllTypesFixture) { + // The committed fixture contains four rows covering values, nulls, and boundary cases. + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_FALSE(fixture.fragment_ids.empty()); + + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("bool_value", TYPE_BOOLEAN, true), + projected_column("tinyint_value", TYPE_TINYINT, true), + projected_column("smallint_value", TYPE_SMALLINT, true), + projected_column("int_value", TYPE_INT, true), + projected_column("bigint_value", TYPE_BIGINT, true), + projected_column("float_value", TYPE_FLOAT, true), + projected_column("double_value", TYPE_DOUBLE, true), + }; + TQueryOptions query_options; + query_options.__set_batch_size(3); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_numeric_types_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + + Block block; + add_output_columns(&block, columns); + std::array seen_rows {}; + const auto read_split = [&](std::vector fragment_ids) { + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, std::move(fragment_ids)).ok()); + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + const auto& bools = + assert_cast(*block.get_by_position(1).column); + const auto& tinyints = + assert_cast(*block.get_by_position(2).column); + const auto& smallints = + assert_cast(*block.get_by_position(3).column); + const auto& ints = assert_cast(*block.get_by_position(4).column); + const auto& bigints = + assert_cast(*block.get_by_position(5).column); + const auto& floats = + assert_cast(*block.get_by_position(6).column); + const auto& doubles = + assert_cast(*block.get_by_position(7).column); + const auto& bool_values = assert_cast(bools.get_nested_column()); + const auto& tinyint_values = + assert_cast(tinyints.get_nested_column()); + const auto& smallint_values = + assert_cast(smallints.get_nested_column()); + const auto& int_values = assert_cast(ints.get_nested_column()); + const auto& bigint_values = + assert_cast(bigints.get_nested_column()); + const auto& float_values = + assert_cast(floats.get_nested_column()); + const auto& double_values = + assert_cast(doubles.get_nested_column()); + for (size_t row = 0; row < block.rows(); ++row) { + const auto row_id = row_ids.get_data()[row]; + ASSERT_GE(row_id, 1); + ASSERT_LE(row_id, 4); + EXPECT_FALSE(seen_rows[row_id]); + seen_rows[row_id] = true; + if (row_id == 3) { + EXPECT_EQ(1, bools.get_null_map_data()[row]); + EXPECT_EQ(1, tinyints.get_null_map_data()[row]); + EXPECT_EQ(1, smallints.get_null_map_data()[row]); + EXPECT_EQ(1, ints.get_null_map_data()[row]); + EXPECT_EQ(1, bigints.get_null_map_data()[row]); + EXPECT_EQ(1, floats.get_null_map_data()[row]); + EXPECT_EQ(1, doubles.get_null_map_data()[row]); + } else { + EXPECT_EQ(0, bools.get_null_map_data()[row]); + EXPECT_EQ(0, tinyints.get_null_map_data()[row]); + EXPECT_EQ(0, smallints.get_null_map_data()[row]); + EXPECT_EQ(0, ints.get_null_map_data()[row]); + EXPECT_EQ(0, bigints.get_null_map_data()[row]); + EXPECT_EQ(0, floats.get_null_map_data()[row]); + EXPECT_EQ(0, doubles.get_null_map_data()[row]); + if (row_id == 1) { + EXPECT_EQ(1, bool_values.get_data()[row]); + EXPECT_EQ(-128, tinyint_values.get_data()[row]); + EXPECT_EQ(-32768, smallint_values.get_data()[row]); + EXPECT_EQ(-2147483648, int_values.get_data()[row]); + EXPECT_EQ(-9223372036854775807LL, bigint_values.get_data()[row]); + EXPECT_FLOAT_EQ(-1.25F, float_values.get_data()[row]); + EXPECT_DOUBLE_EQ(-1.25, double_values.get_data()[row]); + } else if (row_id == 2) { + EXPECT_EQ(0, bool_values.get_data()[row]); + EXPECT_EQ(127, tinyint_values.get_data()[row]); + EXPECT_EQ(32767, smallint_values.get_data()[row]); + EXPECT_EQ(2147483647, int_values.get_data()[row]); + EXPECT_EQ(9223372036854775807LL, bigint_values.get_data()[row]); + EXPECT_FLOAT_EQ(3.5F, float_values.get_data()[row]); + EXPECT_DOUBLE_EQ(3.5, double_values.get_data()[row]); + } else { + EXPECT_EQ(0, bool_values.get_data()[row]); + EXPECT_EQ(0, tinyint_values.get_data()[row]); + EXPECT_EQ(0, smallint_values.get_data()[row]); + EXPECT_EQ(0, int_values.get_data()[row]); + EXPECT_EQ(0, bigint_values.get_data()[row]); + EXPECT_FLOAT_EQ(0.0F, float_values.get_data()[row]); + EXPECT_DOUBLE_EQ(0.0, double_values.get_data()[row]); + } + } + } + } + }; + for (const auto fragment_id : fixture.fragment_ids) { + read_split({fragment_id}); + } + EXPECT_TRUE(seen_rows[1]); + EXPECT_TRUE(seen_rows[2]); + EXPECT_TRUE(seen_rows[3]); + EXPECT_TRUE(seen_rows[4]); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderTypeTest, ReadsDecimalTypesFromAllTypesFixture) { + // The committed fixture contains four rows covering values, nulls, and boundary cases. + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_FALSE(fixture.fragment_ids.empty()); + + const std::array, 8> decimal_specs {{ + {"decimal_1_0", TYPE_DECIMAL32, 1, 0}, + {"decimal_9_2", TYPE_DECIMAL32, 9, 2}, + {"decimal_10_0", TYPE_DECIMAL64, 10, 0}, + {"decimal_18_4", TYPE_DECIMAL64, 18, 4}, + {"decimal_19_0", TYPE_DECIMAL128I, 19, 0}, + {"decimal_38_10", TYPE_DECIMAL128I, 38, 10}, + {"decimal_39_4", TYPE_DECIMAL256, 39, 4}, + {"decimal_76_38", TYPE_DECIMAL256, 76, 38}, + }}; + Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + for (const auto& [name, type, precision, scale] : decimal_specs) { + columns.emplace_back(projected_column(name, nullable_type(type, precision, scale))); + } + + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_decimal_types_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + const std::array, 8> expected_values {{ + {{"-9", "9", "0"}}, + {{"-9999999.99", "9999999.99", "0.00"}}, + {{"-9999999999", "9999999999", "0"}}, + {{"-99999999999999.9999", "99999999999999.9999", "0.0000"}}, + {{"-9999999999999999999", "9999999999999999999", "0"}}, + {{"-9999999999999999999999999999.9999999999", "9999999999999999999999999999.9999999999", + "0.0000000000"}}, + {{"-99999999999999999999999999999999999.9999", + "99999999999999999999999999999999999.9999", "0.0000"}}, + {{"-99999999999999999999999999999999999999.99999999999999999999999999999999999999", + "99999999999999999999999999999999999999.99999999999999999999999999999999999999", + "0.00000000000000000000000000000000000000"}}, + }}; + std::array seen_rows {}; + size_t total_rows = 0; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + for (size_t row = 0; row < block.rows(); ++row) { + const auto row_id = row_ids.get_data()[row]; + ASSERT_GE(row_id, 1); + ASSERT_LE(row_id, 4); + EXPECT_FALSE(seen_rows[row_id]); + seen_rows[row_id] = true; + for (size_t column = 0; column < decimal_specs.size(); ++column) { + const auto& decimal_column = assert_cast( + *block.get_by_position(column + 1).column); + if (row_id == 3) { + EXPECT_EQ(1, decimal_column.get_null_map_data()[row]); + } else { + EXPECT_EQ(0, decimal_column.get_null_map_data()[row]); + const auto expected_index = row_id == 1 ? 0 : row_id == 2 ? 1 : 2; + EXPECT_EQ(expected_values[column][expected_index], + columns[column + 1].type->to_string(decimal_column, row)); + } + } + } + total_rows += block.rows(); + } + EXPECT_EQ(4U, total_rows); + EXPECT_TRUE(seen_rows[1]); + EXPECT_TRUE(seen_rows[2]); + EXPECT_TRUE(seen_rows[3]); + EXPECT_TRUE(seen_rows[4]); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderTypeTest, ReadsStringAndBinaryTypesFromAllTypesFixture) { + // The committed fixture contains four rows covering values, nulls, and boundary cases. + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_FALSE(fixture.fragment_ids.empty()); + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("text_value", TYPE_STRING, true), + projected_column("binary_value", TYPE_VARBINARY, true), + }; + + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_string_binary_types_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + std::array seen_rows {}; + size_t total_rows = 0; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + const auto& texts = assert_cast(*block.get_by_position(1).column); + const auto& binaries = assert_cast(*block.get_by_position(2).column); + for (size_t row = 0; row < block.rows(); ++row) { + const auto row_id = row_ids.get_data()[row]; + ASSERT_GE(row_id, 1); + ASSERT_LE(row_id, 4); + EXPECT_FALSE(seen_rows[row_id]); + seen_rows[row_id] = true; + if (row_id == 3) { + EXPECT_EQ(1, texts.get_null_map_data()[row]); + EXPECT_EQ(1, binaries.get_null_map_data()[row]); + continue; + } + EXPECT_EQ(0, texts.get_null_map_data()[row]); + EXPECT_EQ(0, binaries.get_null_map_data()[row]); + const auto text = texts.get_nested_column().get_data_at(row).to_string(); + const auto binary = binaries.get_nested_column().get_data_at(row).to_string(); + if (row_id == 1) { + EXPECT_EQ("", text); + EXPECT_EQ("", binary); + } else if (row_id == 2) { + EXPECT_EQ("Doris 与 Lance ��", text); + EXPECT_EQ(std::string("\x00\x01\xFF\x7F", 4), binary); + } else { + EXPECT_EQ( + "a moderately long text value used to exercise variable-length Arrow " + "buffers", + text); + EXPECT_EQ("lance", binary); + } + } + total_rows += block.rows(); + } + EXPECT_EQ(4U, total_rows); + EXPECT_TRUE(seen_rows[1]); + EXPECT_TRUE(seen_rows[2]); + EXPECT_TRUE(seen_rows[3]); + EXPECT_TRUE(seen_rows[4]); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderTypeTest, ReadsComplexTypesFromAllTypesFixture) { + // The committed fixture contains four rows covering values, nulls, and boundary cases. + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_FALSE(fixture.fragment_ids.empty()); + + const auto int_type = nullable_type(TYPE_INT); + const auto string_type = nullable_type(TYPE_STRING); + const auto int_array = make_nullable(std::make_shared(int_type)); + const auto attributes = make_nullable(std::make_shared(string_type, int_type)); + const auto profile = make_nullable(std::make_shared( + DataTypes {string_type, int_type}, Strings {"city", "level"})); + const auto visit = make_nullable(std::make_shared( + DataTypes {string_type, int_type}, Strings {"page", "duration_seconds"})); + const auto visits = make_nullable(std::make_shared(visit)); + const auto scores = make_nullable(std::make_shared( + string_type, make_nullable(std::make_shared(int_type)))); + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("int_array", int_array), + projected_column("attributes", attributes), + projected_column("profile", profile), + projected_column("visits", visits), + projected_column("scores_by_source", scores), + }; + + TQueryOptions query_options; + query_options.__set_batch_size(3); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile_stats("lance_complex_types_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile_stats, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + std::array seen_rows {}; + size_t total_rows = 0; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + const auto& int_arrays_column = + assert_cast(*block.get_by_position(1).column); + const auto& attributes_column = + assert_cast(*block.get_by_position(2).column); + const auto& profiles = assert_cast(*block.get_by_position(3).column); + const auto& visits_column = + assert_cast(*block.get_by_position(4).column); + const auto& scores_column = + assert_cast(*block.get_by_position(5).column); + const auto& int_arrays = + assert_cast(int_arrays_column.get_nested_column()); + const auto& int_array_items = assert_cast(int_arrays.get_data()); + const auto& int_array_values = + assert_cast(int_array_items.get_nested_column()); + const auto& attribute_map = + assert_cast(attributes_column.get_nested_column()); + const auto& attribute_keys = assert_cast(attribute_map.get_keys()); + const auto& attribute_key_values = + assert_cast(attribute_keys.get_nested_column()); + const auto& attribute_items = + assert_cast(attribute_map.get_values()); + const auto& attribute_item_values = + assert_cast(attribute_items.get_nested_column()); + const auto& profile_struct = assert_cast(profiles.get_nested_column()); + const auto& profile_cities = + assert_cast(profile_struct.get_column(0)); + const auto& profile_city_values = + assert_cast(profile_cities.get_nested_column()); + const auto& profile_levels = + assert_cast(profile_struct.get_column(1)); + const auto& profile_level_values = + assert_cast(profile_levels.get_nested_column()); + const auto& visits = assert_cast(visits_column.get_nested_column()); + const auto& visit_items = assert_cast(visits.get_data()); + const auto& visit_struct = + assert_cast(visit_items.get_nested_column()); + const auto& visit_pages = assert_cast(visit_struct.get_column(0)); + const auto& visit_page_values = + assert_cast(visit_pages.get_nested_column()); + const auto& visit_durations = + assert_cast(visit_struct.get_column(1)); + const auto& visit_duration_values = + assert_cast(visit_durations.get_nested_column()); + const auto& score_map = assert_cast(scores_column.get_nested_column()); + const auto& score_keys = assert_cast(score_map.get_keys()); + const auto& score_key_values = + assert_cast(score_keys.get_nested_column()); + const auto& score_items = assert_cast(score_map.get_values()); + const auto& score_arrays = assert_cast(score_items.get_nested_column()); + const auto& score_array_items = assert_cast(score_arrays.get_data()); + const auto& score_array_values = + assert_cast(score_array_items.get_nested_column()); + for (size_t row = 0; row < block.rows(); ++row) { + const auto row_id = row_ids.get_data()[row]; + ASSERT_GE(row_id, 1); + ASSERT_LE(row_id, 4); + EXPECT_FALSE(seen_rows[row_id]); + seen_rows[row_id] = true; + if (row_id == 1) { + EXPECT_EQ(0, int_arrays_column.get_null_map_data()[row]); + const auto [int_begin, int_end] = array_range(int_arrays, row); + ASSERT_EQ(3U, int_end - int_begin); + EXPECT_EQ(0, int_array_items.get_null_map_data()[int_begin]); + EXPECT_EQ(1, int_array_values.get_data()[int_begin]); + EXPECT_EQ(1, int_array_items.get_null_map_data()[int_begin + 1]); + EXPECT_EQ(0, int_array_items.get_null_map_data()[int_begin + 2]); + EXPECT_EQ(3, int_array_values.get_data()[int_begin + 2]); + + EXPECT_EQ(0, attributes_column.get_null_map_data()[row]); + const auto [attribute_begin, attribute_end] = map_range(attribute_map, row); + ASSERT_EQ(2U, attribute_end - attribute_begin); + EXPECT_EQ("views", attribute_key_values.get_data_at(attribute_begin).to_string()); + EXPECT_EQ(10, attribute_item_values.get_data()[attribute_begin]); + EXPECT_EQ("likes", + attribute_key_values.get_data_at(attribute_begin + 1).to_string()); + EXPECT_EQ(2, attribute_item_values.get_data()[attribute_begin + 1]); + + EXPECT_EQ(0, profiles.get_null_map_data()[row]); + EXPECT_EQ(0, profile_cities.get_null_map_data()[row]); + EXPECT_EQ("Beijing", profile_city_values.get_data_at(row).to_string()); + EXPECT_EQ(0, profile_levels.get_null_map_data()[row]); + EXPECT_EQ(7, profile_level_values.get_data()[row]); + + EXPECT_EQ(0, visits_column.get_null_map_data()[row]); + const auto [visit_begin, visit_end] = array_range(visits, row); + ASSERT_EQ(2U, visit_end - visit_begin); + EXPECT_EQ(0, visit_items.get_null_map_data()[visit_begin]); + EXPECT_EQ("home", visit_page_values.get_data_at(visit_begin).to_string()); + EXPECT_EQ(0, visit_durations.get_null_map_data()[visit_begin]); + EXPECT_EQ(5, visit_duration_values.get_data()[visit_begin]); + EXPECT_EQ(0, visit_items.get_null_map_data()[visit_begin + 1]); + EXPECT_EQ("search", visit_page_values.get_data_at(visit_begin + 1).to_string()); + EXPECT_EQ(1, visit_durations.get_null_map_data()[visit_begin + 1]); + + EXPECT_EQ(0, scores_column.get_null_map_data()[row]); + const auto [score_begin, score_end] = map_range(score_map, row); + ASSERT_EQ(2U, score_end - score_begin); + EXPECT_EQ("organic", score_key_values.get_data_at(score_begin).to_string()); + const auto [organic_begin, organic_end] = array_range(score_arrays, score_begin); + ASSERT_EQ(2U, organic_end - organic_begin); + EXPECT_EQ(1, score_array_values.get_data()[organic_begin]); + EXPECT_EQ(2, score_array_values.get_data()[organic_begin + 1]); + EXPECT_EQ("ad", score_key_values.get_data_at(score_begin + 1).to_string()); + const auto [ad_begin, ad_end] = array_range(score_arrays, score_begin + 1); + ASSERT_EQ(1U, ad_end - ad_begin); + EXPECT_EQ(3, score_array_values.get_data()[ad_begin]); + } else if (row_id == 2) { + EXPECT_EQ(0, int_arrays_column.get_null_map_data()[row]); + const auto [int_begin, int_end] = array_range(int_arrays, row); + EXPECT_EQ(int_begin, int_end); + + EXPECT_EQ(0, attributes_column.get_null_map_data()[row]); + const auto [attribute_begin, attribute_end] = map_range(attribute_map, row); + EXPECT_EQ(attribute_begin, attribute_end); + + EXPECT_EQ(0, profiles.get_null_map_data()[row]); + EXPECT_EQ(0, profile_cities.get_null_map_data()[row]); + EXPECT_EQ("", profile_city_values.get_data_at(row).to_string()); + EXPECT_EQ(0, profile_levels.get_null_map_data()[row]); + EXPECT_EQ(0, profile_level_values.get_data()[row]); + + EXPECT_EQ(0, visits_column.get_null_map_data()[row]); + const auto [visit_begin, visit_end] = array_range(visits, row); + EXPECT_EQ(visit_begin, visit_end); + + EXPECT_EQ(0, scores_column.get_null_map_data()[row]); + const auto [score_begin, score_end] = map_range(score_map, row); + EXPECT_EQ(score_begin, score_end); + } else { + EXPECT_EQ(1, int_arrays_column.get_null_map_data()[row]); + EXPECT_EQ(1, attributes_column.get_null_map_data()[row]); + EXPECT_EQ(1, profiles.get_null_map_data()[row]); + EXPECT_EQ(1, visits_column.get_null_map_data()[row]); + EXPECT_EQ(1, scores_column.get_null_map_data()[row]); + } + } + total_rows += block.rows(); + } + EXPECT_EQ(4U, total_rows); + EXPECT_TRUE(seen_rows[1]); + EXPECT_TRUE(seen_rows[2]); + EXPECT_TRUE(seen_rows[3]); + EXPECT_TRUE(seen_rows[4]); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderTypeTest, ReadsVectorTypesFromAllTypesFixture) { + // The committed fixture contains four rows covering values, nulls, and boundary cases. + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_FALSE(fixture.fragment_ids.empty()); + + const auto embedding = + make_nullable(std::make_shared(nullable_type(TYPE_FLOAT))); + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("label", TYPE_STRING, true), + projected_column("embedding", embedding), + }; + TQueryOptions query_options; + query_options.__set_batch_size(3); + TQueryGlobals query_globals; + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_vector_types_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + std::array seen_rows {}; + size_t total_rows = 0; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + const auto& labels = assert_cast(*block.get_by_position(1).column); + const auto& label_values = assert_cast(labels.get_nested_column()); + const auto& embeddings = + assert_cast(*block.get_by_position(2).column); + const auto& embedding_arrays = + assert_cast(embeddings.get_nested_column()); + const auto& embedding_items = + assert_cast(embedding_arrays.get_data()); + const auto& embedding_values = + assert_cast(embedding_items.get_nested_column()); + const auto expect_embedding = [&](size_t row, const std::string& label, + const std::array& expected) { + EXPECT_EQ(0, labels.get_null_map_data()[row]); + EXPECT_EQ(label, label_values.get_data_at(row).to_string()); + EXPECT_EQ(0, embeddings.get_null_map_data()[row]); + const auto [begin, end] = array_range(embedding_arrays, row); + ASSERT_EQ(3U, end - begin); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(0, embedding_items.get_null_map_data()[begin + i]); + EXPECT_FLOAT_EQ(expected[i], embedding_values.get_data()[begin + i]); + } + }; + for (size_t row = 0; row < block.rows(); ++row) { + const auto row_id = row_ids.get_data()[row]; + ASSERT_GE(row_id, 1); + ASSERT_LE(row_id, 4); + EXPECT_FALSE(seen_rows[row_id]); + seen_rows[row_id] = true; + if (row_id == 1) { + expect_embedding(row, "origin", {0.0F, 0.0F, 0.0F}); + } else if (row_id == 2) { + expect_embedding(row, "unit-x", {1.0F, 0.0F, 0.0F}); + } else if (row_id == 3) { + expect_embedding(row, "mixed", {-1.5F, 0.25F, 3.75F}); + } else { + expect_embedding(row, "extra", {2.0F, -2.0F, 0.5F}); + } + } + total_rows += block.rows(); + } + EXPECT_EQ(4U, total_rows); + EXPECT_TRUE(seen_rows[1]); + EXPECT_TRUE(seen_rows[2]); + EXPECT_TRUE(seen_rows[3]); + EXPECT_TRUE(seen_rows[4]); + EXPECT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderTypeTest, ReadsTemporalTypesFromAllTypesFixture) { + // The committed fixture contains four rows covering values, nulls, and boundary cases. + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_FALSE(fixture.fragment_ids.empty()); + const Columns columns { + projected_column("row_id", TYPE_BIGINT, false), + projected_column("date_value", TYPE_DATEV2, true), + projected_column("timestamp_value", nullable_type(TYPE_DATETIMEV2, 0, 6)), + }; + + // Spark generated this fixture with spark.sql.session.timeZone=Asia/Shanghai (+08:00). + TimezoneUtils::load_offsets_to_cache(); + TQueryOptions query_options; + query_options.__set_batch_size(4); + TQueryGlobals query_globals; + query_globals.__set_time_zone("+08:00"); + RuntimeState state(TQueryOptions(), query_globals); + state.set_query_options(query_options); + RuntimeProfile profile("lance_temporal_types_fixture"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_fixture(&reader, dataset_uri, fixture, fixture.fragment_ids).ok()); + + Block block; + add_output_columns(&block, columns); + std::array seen_rows {}; + size_t total_rows = 0; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& row_ids = assert_cast(*block.get_by_position(0).column); + const auto& dates = assert_cast(*block.get_by_position(1).column); + const auto& timestamps = + assert_cast(*block.get_by_position(2).column); + for (size_t row = 0; row < block.rows(); ++row) { + const auto row_id = row_ids.get_data()[row]; + ASSERT_GE(row_id, 1); + ASSERT_LE(row_id, 4); + EXPECT_FALSE(seen_rows[row_id]); + seen_rows[row_id] = true; + if (row_id == 1) { + EXPECT_EQ("1969-12-31", columns[1].type->to_string(dates, row)); + EXPECT_EQ("1969-12-31 23:59:59.123456", + columns[2].type->to_string(timestamps, row)); + } else if (row_id == 2) { + EXPECT_EQ("1970-01-01", columns[1].type->to_string(dates, row)); + EXPECT_EQ("1970-01-01 08:00:00.000001", + columns[2].type->to_string(timestamps, row)); + } else if (row_id == 3) { + EXPECT_EQ("2024-02-29", columns[1].type->to_string(dates, row)); + EXPECT_EQ("2024-02-29 12:34:56.654321", + columns[2].type->to_string(timestamps, row)); + } else { + EXPECT_EQ(1, dates.get_null_map_data()[row]); + EXPECT_EQ(1, timestamps.get_null_map_data()[row]); + } + } + total_rows += block.rows(); + } + EXPECT_EQ(4U, total_rows); + EXPECT_TRUE(seen_rows[1]); + EXPECT_TRUE(seen_rows[2]); + EXPECT_TRUE(seen_rows[3]); + EXPECT_TRUE(seen_rows[4]); + EXPECT_TRUE(reader.close().ok()); +} + +} // namespace doris::format::lance diff --git a/build-support/check-header-deps.py b/build-support/check-header-deps.py index 8b2eb403755636..5be3dcab629c15 100755 --- a/build-support/check-header-deps.py +++ b/build-support/check-header-deps.py @@ -587,11 +587,24 @@ "gen_cpp/Types_types.h": 1993, "gen_cpp/segment_v2.pb.h": 1234, "gen_cpp/PaloInternalService_types.h": 1133, - "util/threadpool.h": 988, + # Rebaselined from 988 at master 76fb880330c plus the Lance reader, + # 2026-10-25 (master alone: 1084). The reader adds four TUs, two sources + # and their tests, and no new edge: they reach it through + # exprs/vexpr_context.h, which holds segment_v2::ColumnIteratorOptions by + # value and so includes storage/segment/column_reader.h -> + # io/cache/cached_remote_file_reader.h -> io/cache/block_file_cache.h. + # Cutting that edge would bring this down to about 846. + "util/threadpool.h": 1088, # parsed_page.h holds RleDecoder by value and constructs it in the # header, so the whole storage read stack sees the RLE machinery; a live # dependency, budgeted as-is (trimming it is a refactor, not a gate). - "util/rle_encoding.h": 823, + # Rebaselined from 823 at master 9af1840dcd0 plus the Lance reader, + # 2026-10-26 (master alone: 903). The reader's four TUs are the ones + # counted under util/threadpool.h above and add no new edge either: they + # reach it through exprs/vexpr_context.h -> storage/segment/column_reader.h + # -> storage/segment/parsed_page.h, so cutting that same edge would bring + # this down too. + "util/rle_encoding.h": 907, "gen_cpp/internal_service.pb.h": 741, "gen_cpp/FrontendService_types.h": 576, "storage/options.h": 464, diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_transactions/0-4d80709b-dcb0-4dea-8c7f-99828f67f781.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_transactions/0-4d80709b-dcb0-4dea-8c7f-99828f67f781.txn new file mode 100644 index 00000000000000..1145f333b55bc7 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_transactions/0-4d80709b-dcb0-4dea-8c7f-99828f67f781.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..19579561068e06 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_versions/18446744073709551614.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000000000..491d734467aa3c --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/data/0000001100111011100111105b710547dc82de2df5345c7ab9.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/data/0000001100111011100111105b710547dc82de2df5345c7ab9.lance new file mode 100644 index 00000000000000..7816ac4a25279d Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/all_types.lance/data/0000001100111011100111105b710547dc82de2df5345c7ab9.lance differ diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index b0da58fdf17dfd..c0baef819c3ff7 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -829,7 +829,9 @@ message PRequestBlockDesc { repeated PSlotDescriptor slots = 2; repeated ColumnPB column_descs = 3; repeated uint32 file_id = 4; - repeated uint32 row_id = 5; + // The wire field is uint64 for every version. FILE_LOCAL_ROW_ID values are zero-extended + // from uint32, while LANCE_DATASET_ROW_ID uses the full uint64 range. + repeated uint64 row_id = 5; optional PTupleDescriptor desc = 6; repeated uint32 column_idxs = 7; // Numeric value of ROW_VERSION. Keep this as uint32 so a receiver can detect and reject a diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 1cb49705c5bd4d..7ccac23cc0c199 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -475,6 +475,126 @@ struct TRemoteDorisFileDesc { 6: optional string password } +// The external search enums below spell out their values because those values are on the wire: +// branch-4.1/4.2 declared the same enums without explicit values, so each member must keep its +// implicit position (FLOAT16 = 0, FLOAT32 = 1, ...) on every branch. + +// Element type and byte layout of a vector search query. The encoded values use little-endian +// byte order and contain exactly `dimension` elements of `element_type`. +enum TVectorElementType { + FLOAT16 = 0, + FLOAT32 = 1, + FLOAT64 = 2, + UINT8 = 3, + INT8 = 4 +} + +enum TVectorMetric { + DEFAULT = 0, + L2 = 1, + COSINE = 2, + DOT_PRODUCT = 3, + HAMMING = 4 +} + +struct TSearchVector { + 1: optional TVectorElementType element_type + 2: optional i32 dimension + 3: optional binary values + // Present only for one multi-vector query; values contains a row-major matrix. + 4: optional i32 num_vectors +} + +// Logical result parameters for one vector query. `top_k` is the number of rows returned after +// skipping `offset`; an adapter may fetch more physical candidates to implement that contract. +struct TVectorSearchParams { + 1: optional string column + 2: optional TSearchVector query_vector + 3: optional i64 top_k + 4: optional i64 offset + 5: optional TVectorMetric metric + // FLOAT bounds: inclusive lower and exclusive upper. + 6: optional double distance_lower_bound + 7: optional double distance_upper_bound +} + +enum TFtsCoverageMode { + STRICT = 0, + INDEX_ONLY = 1 +} + +enum TFtsQueryType { + MATCH = 0, + PHRASE = 1 +} + +enum TFtsMatchOperator { + OR = 0, + AND = 1 +} + +// Logical parameters for one full-text query. MATCH combines analyzed terms with match_operator; +// PHRASE searches the analyzed terms in order and permits phrase_slop intervening positions. +struct TFullTextSearchParams { + 1: optional string column + 2: optional string query + 3: optional i64 top_k + 4: optional i64 offset + // STRICT requires the selected FTS index to cover the complete pinned snapshot. INDEX_ONLY + // searches and scores only fragments covered by committed FTS index segments. + 5: optional TFtsCoverageMode coverage_mode + // Reserved for distributed FTS. Every BE may search a different index-segment subset, so + // statistics prepared independently on each BE would produce BM25 scores that are not + // comparable during the final TopK merge. The payload must be prepared once for the exact + // snapshot, logical FTS index, segment set, and query (including its final fuzzy vocabulary), + // then validated and installed unchanged on every BE scanner. + // + // Doris currently leaves this field unset and BE rejects a set value because Lance/lance-c + // does not yet provide the complete producer and consumer contract. Track the upstream work + // at https://github.com/lance-format/lance/issues/8937. + 6: optional binary global_statistics + 7: optional TFtsQueryType query_type + // MATCH-only parameters. max_fuzzy_distance is reserved as zero until the bundled Lance-C + // supports one canonical fuzzy vocabulary across all prepared index segments. + 8: optional TFtsMatchOperator match_operator + 9: optional i32 max_fuzzy_distance + // PHRASE-only parameter. Zero requires an exact phrase. + 10: optional i32 phrase_slop +} + +enum TSearchFilterFormat { + SQL = 0, + SUBSTRAIT = 1 +} + +// A TVF search filter is evaluated by the search provider. TExternalSearchRequest.prefilter +// controls whether Lance applies it before or after vector/FTS candidate selection. A normal SQL +// predicate above the search relation remains a Doris residual. +struct TSearchFilter { + 1: optional TSearchFilterFormat format + 2: optional binary payload +} + +// Optional vector search tuning. Logical query fields stay in TVectorSearchParams so providers +// can reuse the same request and interpret the supported tuning fields independently. +struct TVectorSearchOptions { + 1: optional i32 nprobes + 2: optional i32 refine_factor + 3: optional i32 ef + 4: optional bool use_index + // Lance: -1 uses available CPU parallelism, 0 selects automatically, positive values cap it. + 5: optional i32 query_parallelism +} + +// The active union field identifies the logical search kind. A future hybrid field can contain both +// vector and full-text subqueries plus its fusion parameters without changing either existing field. +union TExternalSearchQuery { + 1: TVectorSearchParams vector_search + 2: TFullTextSearchParams full_text_search +} + +// A catalog/S3 range reads fragments from a fixed snapshot. A local TVF range uses version zero +// without fragment_ids to open the latest snapshot and scan the whole dataset on its selected BE. struct TLanceFileDesc { // URI of the Lance dataset (s3://..., file:///..., etc.) 1: optional string dataset_uri @@ -482,6 +602,19 @@ struct TLanceFileDesc { 2: optional list fragment_ids // Dataset version for time travel 3: optional i64 version + // Per-split row limit pushed down from the query LIMIT. Each scanner returns at + // most this many rows; the upper LIMIT operator still enforces the global bound. + // Only set for ordinary scans whose predicates are fully pushed into Lance. + 4: optional i64 limit + // Physical index segments assigned to this split. Each UUID is 16 bytes in RFC 4122 order. + // An ordinary scan accepts exactly one scalar segment and requires a fixed version and + // nonempty fragment_ids. Vector/FTS scans interpret these as their own index segments. + // Unset for fragment scans without an assigned segment. + 5: optional list index_segment_uuids + // Ordinary scans only. False for uncovered-fragment tasks in a scalar segment plan, + // so these tasks filter their rows without repeating global scalar-index evaluation. + // Unset preserves Lance's default; an explicit scalar segment must not be combined with false. + 6: optional bool use_scalar_index } struct TTableFormatFileDesc { @@ -530,11 +663,30 @@ enum TTextSerdeType { // provider FileDesc (for example, dataset_uri/version/fragment_ids in TLanceFileDesc). struct TExternalSearchRequest { 1: optional i32 schema_version = 1 + 2: optional TExternalSearchQuery search_query + 3: optional TSearchFilter search_filter + 4: optional TVectorSearchOptions vector_search_options + // Applies only to search_filter. Defaults to false (postfilter); Doris WHERE predicates remain + // residuals regardless of this setting. + 5: optional bool prefilter } struct TLanceScanParams { + // Serialized Substrait ExtendedExpression executed by the native Lance scanner. Set at + // ScanNode level so it is not serialized once per fragment split. 1: optional binary lance_substrait_filter + // Provider-independent search request. Set at ScanNode level so all ranges use the same logical + // query. Lance external search uses one range per fragment or physical index segment, and Doris + // merges the split-local candidates. 2: optional TExternalSearchRequest external_search_request + // Lance-native storage options, handed to lance-c untranslated. The namespace protocol treats + // storage_options as opaque configuration passed directly to Lance, so any key vocabulary the + // BE imposed here would drop options it does not happen to know - including credentials a + // namespace spelled with a different accepted alias, and every non-S3 provider's keys. + // Set at ScanNode level so credentials are not serialized once per fragment split. + // These are the initial options for the scan and are never refreshed: lance-c opens datasets + // with a static option set, so credentials that expire mid-scan are not re-vended. Renewal + // needs a refresh channel of its own, which this field is not. 3: optional map lance_storage_options }